00000800:00000200:1.1F:1767315035.639564:0:7011:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000916fd0ec] -> 12345-192.168.204.126@tcp (2) 00000800:00000200:3.1F:1767315035.640393:0:7004:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000916fd0ec] -> 12345-192.168.204.126@tcp (2) 00000800:00000200:3.1:1767315035.640402:0:7004:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.204.126@tcp 000000007d6ccfe1 deleted 00000800:00000010:3.1:1767315035.640405:0:7004:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000007d6ccfe1. 00000800:00000200:3.1:1767315035.640409:0:7004:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1767315035.640415:0:7004:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.126@tcp ip 192.168.204.126:1023 00000800:00000200:3.1:1767315035.640420:0:7004:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000bde00799 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:3.1:1767315035.640591:0:7002:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.1F:1767315035.642706:0:7003:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000916fd0ec] -> 12345-192.168.204.126@tcp (4) 00000800:00000200:2.1:1767315035.642714:0:7003:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.204.126@tcp 000000009ce83652 deleted 00000800:00000010:2.1:1767315035.642716:0:7003:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 000000009ce83652. 00000800:00000200:2.1:1767315035.642719:0:7003:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1 00000800:00000200:3.1:1767315035.647588:0:7002:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1 00010000:00000010:2.1:1767315038.809560:0:15443:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000eb915a3f. 00010000:00000010:2.1:1767315038.809567:0:15443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000b3b4a0b3. 00010000:00000010:0.1F:1767315039.009199:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000765cc6a9. 00010000:00000010:0.1:1767315039.009216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000cb9f4587. 00010000:00000010:0.1:1767315039.009220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000074f6f31c. 00010000:00000010:0.1:1767315039.009223:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000ac0fe3e4. 00010000:00000010:0.1:1767315039.009226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000097099c26. 00010000:00000010:2.1:1767315039.133352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000004413da41. 00010000:00000010:0.1:1767315039.134327:0:15434:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000054f2408b. 00010000:00000010:0.1:1767315039.134333:0:15434:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000053195fbc. 00010000:00000010:0.1:1767315039.134337:0:15434:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000b6defa73. 00010000:00000010:0.1:1767315039.134341:0:15434:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000d76be081. 00010000:00000010:3.1:1767315039.135567:0:15512:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000d39f3fdc. 00010000:00000010:3.1:1767315039.135573:0:15512:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000cec3f55c. 00010000:00000010:0.1:1767315039.145643:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000b4f3aec7. 00010000:00000010:0.1:1767315039.145652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000f6f66dc6. 00010000:00000010:0.1:1767315039.545523:0:15539:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 0000000088cd63fb. 00010000:00000010:0.1:1767315039.545536:0:15539:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000f40c07ca. 00010000:00000010:0.1:1767315039.545542:0:15539:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000990a1c4d. 00010000:00000010:0.1:1767315039.545545:0:15539:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000e3b2edfe. 00010000:00000010:0.1:1767315039.545548:0:15539:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000bbe501b1. 00080000:00000010:3.1:1767315039.604757:0:15434:0:(osd_handler.c:1617:osd_oxc_free()) kfreed 'oxe': 68 at 00000000b0ce921d. 00010000:00080000:1.2F:1767315387.240279:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:3.2F:1767315537.240230:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:0.2F:1767315708.240173:0:1045:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:3.2:1767315876.240235:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:0.2:1767316037.240295:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:2.2F:1767316192.240211:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:1.2:1767316343.240201:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:2.2:1767316511.240224:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:3.2:1767316682.240210:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:1.2:1767316817.240217:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:0.2:1767316955.240279:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:2.2:1767317110.240296:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:3.2:1767317252.240210:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:2.2:1767317401.240219:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:0.2:1767317521.240223:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:1.2:1767317642.240212:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:3.2:1767317778.240237:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:2.2:1767317903.241507:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00010000:00080000:2.2:1767318024.241234:0:0:0:(ldlm_lib.c:3011:target_recovery_expired()) lustre-MDT0000: recovery timed out; 1 clients are still in recovery after 70 seconds (2 clients connected) 00000100:00100000:1.0:1767318573.278800:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984948352:12345-0@lo:38:kworker.0 Request processed in 31us (1334us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318573.278807:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4398 00000100:00100000:1.0:1767318573.278820:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4399 00000100:00100000:1.0:1767318573.278824:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984948608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318573.278832:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318573.278854:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984948608:12345-0@lo:38:kworker.0 Request processed in 29us (639us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318573.278859:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4399 00000100:00080000:1.0:1767318573.278912:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318573.278916:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318573.278919:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984947968:0@lo:38:kworker.0 00000100:00080000:1.0:1767318573.278927:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318573.278930:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318573.278932:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984948352:0@lo:38:kworker.0 00000100:00080000:1.0:1767318573.278939:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318573.278941:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318573.278944:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984948608:0@lo:38:kworker.0 00000100:00080000:1.0:1767318574.251331:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318574.256890:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00010000:00010000:1.0:1767318574.942360:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318576.987838:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318577.883642:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984948736:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318577.883715:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318577.884192:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318577.884227:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984948736:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318577.886871:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318578.395538:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318578.395548:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318578.395551:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318578.395555:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318578.395559:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318578.395568:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3821 00000100:00080000:1.0:1767318578.395574:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767318578.395581:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318578.395634:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5c00 x1853163984948992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318578.395661:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318578.395664:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318578.395666:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318578.395670:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3821 00000100:00080000:1.0:1767318578.395672:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767318578.395676:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318578.395690:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4700 x1853163984949120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318578.395722:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318578.395725:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318578.395727:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318578.395731:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3821 00000100:00080000:1.0:1767318578.395733:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767318578.395737:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318578.395749:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7b80 x1853163984949376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318578.395804:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318578.395808:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318578.395810:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318578.395814:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3821 00000100:00080000:1.0:1767318578.395817:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767318578.395820:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318578.395832:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7480 x1853163984949632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318578.395853:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318578.395865:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163984949248:0@lo:400:kworker.0 00000100:00100000:1.0:1767318578.395943:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318578.396023:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163984949504:0@lo:400:kworker.0 00000100:00100000:1.0:1767318578.396046:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318579.382539:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00010000:00010000:1.0:1767318581.083857:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318583.003498:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984949760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318583.003608:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318583.005890:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984949760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318583.005907:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984949888:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318583.005960:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318583.007120:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984949888:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318583.131719:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318584.495264:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318584.495382:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00010000:00010000:1.0:1767318587.227433:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318588.124769:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090000e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984950912:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318588.587342:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767318588.587375:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920512640 offset 0 length 520: 4 00000100:00100000:1.0:1767318588.587447:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318588.635888:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163984951424:0@lo:400:kworker.0 00000100:00100000:1.0:1767318588.635948:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318588.636020:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318588.636026:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984951680:0@lo:400:kworker.0 00000100:00100000:1.0:1767318588.636049:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318588.636080:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984951424 00000100:00100000:1.0:1767318588.636088:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984951424 00000100:00100000:1.0:1767318588.636132:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984951680 00000100:00100000:1.0:1767318588.636135:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984951680 00000100:00100000:1.0:1767318588.636153:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4106 00000100:00100000:1.0:1767318588.636161:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163984951424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318588.636223:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163984951424:12345-0@lo:400:kworker.0 Request processed in 63us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318588.636228:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4106 00000100:00100000:1.0:1767318588.636241:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4107 00000100:00100000:1.0:1767318588.636244:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984951680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318588.636259:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984951680:12345-0@lo:400:kworker.0 Request processed in 15us (210us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318588.636265:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4107 00000100:00100000:1.0:1767318588.636285:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984951680:0@lo:400:kworker.0 00000100:00100000:1.0:1767318588.636290:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163984951424:0@lo:400:kworker.0 00010000:00010000:1.0:1767318589.279104:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318593.243636:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318593.243663:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce090000e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984951936:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318593.243730:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318593.244152:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318593.244498:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318593.376221:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318593.712970:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318593.713025:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767318593.713451:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318598.363432:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318598.364623:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984953216:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318598.364659:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318598.364847:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318598.365013:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984953216:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318598.825865:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767318598.826424:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318598.827721:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318598.876700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984953472:0@lo:38:kworker.0 00000100:00100000:1.0:1767318598.876741:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318598.876764:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984953600:0@lo:38:kworker.0 00000100:00100000:1.0:1767318598.876789:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318598.876813:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ada680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984953856:0@lo:38:kworker.0 00000100:00100000:1.0:1767318598.876833:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318598.876846:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984954112:0@lo:38:kworker.0 00000100:00100000:1.0:1767318598.876863:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318598.876892:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984953600 00000100:00100000:1.0:1767318598.876896:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984953600 00000100:00100000:1.0:1767318598.876919:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1595 00000100:00100000:1.0:1767318598.876923:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984953600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318598.876943:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318598.876982:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984953600:12345-0@lo:38:kworker.0 Request processed in 57us (191us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318598.876990:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1595 00000100:00100000:1.0:1767318598.877013:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984953472 00000100:00100000:1.0:1767318598.877016:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984953472 00000100:00100000:1.0:1767318598.877031:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984953856 00000100:00100000:1.0:1767318598.877034:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984953856 00000100:00100000:1.0:1767318598.877045:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984954112 00000100:00100000:1.0:1767318598.877047:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984954112 00000100:00100000:1.0:1767318598.877060:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4418 00000100:00100000:1.0:1767318598.877064:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984953472:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318598.877074:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318598.877101:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984953472:12345-0@lo:38:kworker.0 Request processed in 35us (359us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318598.877107:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4418 00000100:00100000:1.0:1767318598.877118:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4419 00000100:00100000:1.0:1767318598.877122:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984953856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318598.877130:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318598.877150:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984953856:12345-0@lo:38:kworker.0 Request processed in 27us (316us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318598.877155:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4419 00000100:00100000:1.0:1767318598.877182:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4420 00000100:00100000:1.0:1767318598.877186:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984954112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318598.877193:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318598.877217:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163984954112:12345-0@lo:38:kworker.0 Request processed in 29us (352us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318598.877222:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4420 00000100:00080000:1.0:1767318598.877256:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318598.877261:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318598.877265:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984953472:0@lo:38:kworker.0 00000100:00080000:1.0:1767318598.877274:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318598.877277:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318598.877280:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984953600:0@lo:38:kworker.0 00000100:00080000:1.0:1767318598.877287:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318598.877290:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318598.877292:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ada680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984953856:0@lo:38:kworker.0 00000100:00080000:1.0:1767318598.877299:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318598.877301:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318598.877304:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984954112:0@lo:38:kworker.0 00000100:00100000:1.0:1767318598.877347:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2728 00000100:00100000:1.0:1767318598.877355:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984954368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318598.877407:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984954368:12345-0@lo:400:kworker.0 Request processed in 54us (1004us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318598.877413:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2728 00000100:00100000:1.0:1767318598.877456:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984954368:0@lo:400:kworker.0 00010000:00010000:1.0:1767318599.515409:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318603.483788:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318603.483811:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984954496:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318603.483882:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318603.484386:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318603.484441:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984954496:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318603.948079:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318603.954758:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920513664 00000100:00100000:1.0:1767318603.954768:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920513664 00000100:00100000:1.0:1767318603.954874:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4114 00000100:00100000:1.0:1767318603.954885:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce18b156c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920513664:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318603.954980:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce18b156c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920513664:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 96us (474us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318603.954988:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4114 00010000:00010000:1.0:1767318605.661606:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318608.604559:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984955776 00000100:00100000:1.0:1767318608.604579:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984955776 00000100:00100000:1.0:1767318608.604651:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1880 00000100:00100000:1.0:1767318608.604661:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984955776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318608.604690:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318608.604749:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984955776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (999us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318608.604759:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1880 00000100:00100000:1.0:1767318609.119405:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163984956288:0@lo:400:kworker.0 00000100:00100000:1.0:1767318609.119447:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318609.119492:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984956032 00000100:00100000:1.0:1767318609.119501:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984956032 00000100:00100000:1.0:1767318609.119540:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984956288 00000100:00100000:1.0:1767318609.119543:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984956288 00000100:00100000:1.0:1767318609.119593:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4118 00000100:00100000:1.0:1767318609.119603:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163984956288:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318609.119663:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163984956288:12345-0@lo:400:kworker.0 Request processed in 61us (214us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318609.119671:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4118 00000100:00100000:1.0:1767318609.119691:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4426 00000100:00100000:1.0:1767318609.119695:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984956032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318609.119719:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318609.119755:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984956032:12345-0@lo:38:kworker.0 Request processed in 57us (933us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318609.119761:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4426 00000100:00080000:1.0:1767318609.144448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318609.144462:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318609.144481:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984956032:0@lo:38:kworker.0 00000100:00100000:1.0:1767318609.144500:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984956160:0@lo:38:kworker.0 00000100:00100000:1.0:1767318609.144592:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318609.144669:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984956416:0@lo:38:kworker.0 00000100:00100000:1.0:1767318609.144702:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318609.144725:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984956672:0@lo:38:kworker.0 00000100:00100000:1.0:1767318609.144749:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318609.144817:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984956416 00000100:00100000:1.0:1767318609.144825:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984956416 00000100:00100000:1.0:1767318609.144862:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984956672 00000100:00100000:1.0:1767318609.144866:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984956672 00000100:00100000:1.0:1767318609.144887:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4427 00000100:00100000:1.0:1767318609.144896:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984956416:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318609.144946:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318609.144998:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984956416:12345-0@lo:38:kworker.0 Request processed in 101us (293us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318609.145006:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4427 00000100:00100000:1.0:1767318609.145027:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4428 00000100:00100000:1.0:1767318609.145031:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984956672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318609.145041:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318609.145066:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984956672:12345-0@lo:38:kworker.0 Request processed in 34us (316us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318609.145072:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4428 00000100:00100000:1.0:1767318609.145100:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984956160 00000100:00100000:1.0:1767318609.145103:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984956160 00000100:00100000:1.0:1767318609.145124:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1597 00000100:00100000:1.0:1767318609.145128:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984956160:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318609.145137:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318609.145193:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984956160:12345-0@lo:38:kworker.0 Request processed in 35us (573us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318609.145203:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1597 00000100:00100000:1.0:1767318609.145250:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163984956288:0@lo:400:kworker.0 00000100:00080000:1.0:1767318609.145289:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318609.145293:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318609.145296:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984956160:0@lo:38:kworker.0 00000100:00080000:1.0:1767318609.145304:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318609.145307:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318609.145310:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984956416:0@lo:38:kworker.0 00000100:00080000:1.0:1767318609.145318:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318609.145321:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318609.145324:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984956672:0@lo:38:kworker.0 00010000:00010000:1.0:1767318611.806237:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318613.275871:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318588/real 1767318588] req@ffff8ce1a6ad9500 x1853163984951040/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318613 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318613.275906:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a6ad9500 x1853163984951040/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318613 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318613.275924:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318613.275953:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318613.275957:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163984951040:0@lo:250:kworker.0 00000100:00100000:1.0:1767318613.284406:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163984957056:0@lo:250:kworker.0 00000400:00000100:1.0:1767318613.284469:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163984957056 offset 0 length 520: 4 00000100:00100000:1.0:1767318613.284886:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984957184:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.284925:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318613.285376:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984957696:0@lo:400:kworker.0 00000100:00100000:1.0:1767318613.285424:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984957440 00000100:00100000:1.0:1767318613.285428:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984957440 00000100:00100000:1.0:1767318613.285463:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984957696 00000100:00100000:1.0:1767318613.285466:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984957696 00000100:00100000:1.0:1767318613.285489:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4121 00000100:00100000:1.0:1767318613.285496:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984957696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318613.285527:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984957696:12345-0@lo:400:kworker.0 Request processed in 33us (86us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318613.285534:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4121 00000100:00100000:1.0:1767318613.285568:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984957696:0@lo:400:kworker.0 00000100:00080000:1.0:1767318613.286396:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318613.286400:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318613.286420:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984957312:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.286448:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318613.286468:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984957568:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.286490:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318613.286504:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984957824:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.286523:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318613.286546:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318613.286551:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318613.286555:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984957184:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.286579:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984957568 00000100:00100000:1.0:1767318613.286581:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984957568 00000100:00100000:1.0:1767318613.286598:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984957824 00000100:00100000:1.0:1767318613.286600:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984957824 00000100:00100000:1.0:1767318613.286613:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4430 00000100:00100000:1.0:1767318613.286618:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984957568:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318613.286627:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318613.286649:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984957568:12345-0@lo:38:kworker.0 Request processed in 30us (158us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318613.286674:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4430 00000100:00080000:1.0:1767318613.288361:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318613.288375:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318613.288381:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984957312:0@lo:38:kworker.0 00000100:00080000:1.0:1767318613.288397:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318613.288401:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318613.288404:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984957568:0@lo:38:kworker.0 00000100:00080000:1.0:1767318613.288417:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318613.288421:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318613.288423:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984957824:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.723734:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318613.723760:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984958208:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318613.723834:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318613.724822:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984958336 00000100:00100000:1.0:1767318613.724827:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984958336 00000100:00100000:1.0:1767318613.724899:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1883 00000100:00100000:1.0:1767318613.724907:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e374380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984958336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318613.724933:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318613.725232:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e374380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984958336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 325us (561us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318613.725239:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1883 00000100:00100000:1.0:1767318614.190818:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920514304 00000100:00100000:1.0:1767318614.190833:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920514304 00000100:00100000:1.0:1767318614.190916:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920514176 00000100:00100000:1.0:1767318614.190920:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920514176 00000100:00100000:1.0:1767318614.190942:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4122 00000100:00100000:1.0:1767318614.190950:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920514304:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318614.191239:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920514304:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (2313us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318614.191247:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4122 00000100:00100000:1.0:1767318614.191264:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4432 00000100:00100000:1.0:1767318614.191267:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920514176:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318614.191285:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318614.191307:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920514176:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 39us (2539us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318614.191311:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4432 00010000:00010000:1.0:1767318617.948095:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318618.331953:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984958464:0@lo:38:kworker.0 00000100:00100000:1.0:1767318618.353507:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318618.353547:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984958592:0@lo:38:kworker.0 00000100:00100000:1.0:1767318618.353582:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318618.354370:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984958848:0@lo:38:kworker.0 00000100:00100000:1.0:1767318618.354405:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318618.354423:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984959104:0@lo:38:kworker.0 00000100:00100000:1.0:1767318618.354442:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318618.354469:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984958464 00000100:00100000:1.0:1767318618.354475:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984958464 00000100:00100000:1.0:1767318618.354498:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984958848 00000100:00100000:1.0:1767318618.354501:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984958848 00000100:00100000:1.0:1767318618.354525:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984959104 00000100:00100000:1.0:1767318618.354528:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984959104 00000100:00100000:1.0:1767318618.354545:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4433 00000100:00100000:1.0:1767318618.354552:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984958464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318618.354573:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318618.354611:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984958464:12345-0@lo:38:kworker.0 Request processed in 58us (1103us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318618.354618:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4433 00000100:00100000:1.0:1767318618.354632:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4434 00000100:00100000:1.0:1767318618.354635:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984958848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318618.354643:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318618.354665:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984958848:12345-0@lo:38:kworker.0 Request processed in 27us (259us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318618.354670:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4434 00000100:00100000:1.0:1767318618.354679:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4435 00000100:00100000:1.0:1767318618.354683:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984959104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318618.354902:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318618.354929:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984959104:12345-0@lo:38:kworker.0 Request processed in 245us (486us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318618.354947:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4435 00000100:00080000:1.0:1767318618.354983:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318618.354988:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318618.354991:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984959104:0@lo:38:kworker.0 00000100:00080000:1.0:1767318618.355021:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318618.355026:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318618.355028:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984958464:0@lo:38:kworker.0 00000100:00080000:1.0:1767318618.355036:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318618.355040:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318618.355042:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984958848:0@lo:38:kworker.0 00000100:00100000:1.0:1767318618.843645:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984959488 00000100:00100000:1.0:1767318618.843663:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984959488 00000100:00100000:1.0:1767318618.843714:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1884 00000100:00100000:1.0:1767318618.843723:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984959488:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318618.843751:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318618.854333:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984959488:12345-0@lo:13:osp-pre-0-1.0 Request processed in 10601us (10832us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318618.854349:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1884 00000100:00080000:1.0:1767318619.308361:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318623.459953:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318623.459967:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318623.459972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318623.461205:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984960000:0@lo:400:kworker.0 00000100:00100000:1.0:1767318623.963655:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984960768 00000100:00100000:1.0:1767318623.963675:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984960768 00000100:00100000:1.0:1767318623.963771:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1886 00000100:00100000:1.0:1767318623.963787:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090001880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984960768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318623.963837:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318623.963934:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090001880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984960768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 152us (337us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318623.963944:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1886 00000100:00100000:1.0:1767318623.963982:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984960896:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318623.964033:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318623.964148:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984960768:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318624.094056:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318624.434372:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318624.434439:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767318624.434905:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318628.573965:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318628.573971:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318628.573973:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318628.574420:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984961024 00000100:00100000:1.0:1767318628.574425:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984961024 00000100:00100000:1.0:1767318628.574465:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984961408 00000100:00100000:1.0:1767318628.574469:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984961408 00000100:00100000:1.0:1767318628.574484:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984961664 00000100:00100000:1.0:1767318628.574486:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984961664 00000100:00100000:1.0:1767318628.574497:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4441 00000100:00100000:1.0:1767318628.574503:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984961024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318628.574516:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318628.574567:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984961024:12345-0@lo:38:kworker.0 Request processed in 63us (1102us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318628.574574:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4441 00000100:00100000:1.0:1767318628.574586:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4442 00000100:00100000:1.0:1767318628.574590:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984961408:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318628.574600:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318628.574623:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984961408:12345-0@lo:38:kworker.0 Request processed in 31us (1061us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318628.574629:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4442 00000100:00100000:1.0:1767318628.574640:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4443 00000100:00100000:1.0:1767318628.574644:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984961664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318628.574652:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318628.574676:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984961664:12345-0@lo:38:kworker.0 Request processed in 31us (1079us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318628.574682:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4443 00000100:00080000:1.0:1767318628.574707:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318628.574712:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318628.574716:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090000700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984961024:0@lo:38:kworker.0 00000100:00080000:1.0:1767318628.574725:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318628.574729:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318628.574732:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090000000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984961408:0@lo:38:kworker.0 00000100:00080000:1.0:1767318628.574739:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318628.574742:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318628.574745:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090003b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984961664:0@lo:38:kworker.0 00000100:00100000:1.0:1767318629.547865:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920515072 00000100:00100000:1.0:1767318629.547875:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920515072 00000100:00100000:1.0:1767318629.547942:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920514944 00000100:00100000:1.0:1767318629.547945:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920514944 00000100:00100000:1.0:1767318629.547964:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4131 00000100:00100000:1.0:1767318629.547974:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920515072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318629.548049:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920515072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (527us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318629.548057:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4131 00000100:00100000:1.0:1767318629.548338:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4444 00000100:00100000:1.0:1767318629.548347:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920514944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318629.548367:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318629.548395:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920514944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (808us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318629.550843:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4444 00010000:00010000:1.0:1767318630.235475:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318633.692768:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984962304:0@lo:38:kworker.0 00000100:00100000:1.0:1767318633.692844:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318633.692919:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984962432:0@lo:38:kworker.0 00000100:00100000:1.0:1767318633.692960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318633.692995:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984962688:0@lo:38:kworker.0 00000100:00100000:1.0:1767318633.693022:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318633.693047:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984962944:0@lo:38:kworker.0 00000100:00100000:1.0:1767318633.693073:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318633.693107:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984962560 00000100:00100000:1.0:1767318633.693113:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984962560 00000100:00100000:1.0:1767318633.693148:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4132 00000100:00100000:1.0:1767318633.693158:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984962816:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318633.693243:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984962816:12345-0@lo:400:kworker.0 Request processed in 85us (327us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318633.693874:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4132 00000100:00100000:1.0:1767318633.693897:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4446 00000100:00100000:1.0:1767318633.693903:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984962688:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318633.693917:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318633.693957:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984962688:12345-0@lo:38:kworker.0 Request processed in 50us (932us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318633.693967:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4446 00000100:00080000:1.0:1767318633.694006:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318633.694012:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318633.694017:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984962688:0@lo:38:kworker.0 00000100:00080000:1.0:1767318633.694030:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318633.694037:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318633.694042:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984962944:0@lo:38:kworker.0 00000100:00080000:1.0:1767318633.694052:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318633.694055:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318633.694058:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984962304:0@lo:38:kworker.0 00000100:00100000:1.0:1767318633.694101:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984962432 00000100:00100000:1.0:1767318633.694107:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984962432 00000100:00100000:1.0:1767318633.694132:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1602 00000100:00100000:1.0:1767318633.694138:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984962432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318633.694148:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318633.694202:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984962432:12345-0@lo:38:kworker.0 Request processed in 62us (1242us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318633.694213:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1602 00000100:00080000:1.0:1767318633.694241:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318633.694245:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318633.694248:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984962432:0@lo:38:kworker.0 00000100:00080000:1.0:1767318634.667125:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318636.387105:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318638.767272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318638.814137:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984963712 00000100:00100000:1.0:1767318638.814143:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984963712 00000100:00100000:1.0:1767318638.814201:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1603 00000100:00100000:1.0:1767318638.814208:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984963712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318638.814231:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318638.814274:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984963712:12345-0@lo:38:kworker.0 Request processed in 65us (365us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318638.814280:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1603 00000100:00100000:1.0:1767318639.323912:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318639.323945:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984964736:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318639.324044:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318639.332339:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984964864 00000100:00100000:1.0:1767318639.332359:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984964864 00000100:00100000:1.0:1767318639.332462:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1893 00000100:00100000:1.0:1767318639.332476:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984964864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318639.332524:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318639.337158:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984964864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4669us (9828us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318639.337195:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1893 00010000:00010000:1.0:1767318642.523957:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318643.931794:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318643.936142:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984965248:0@lo:400:kworker.0 00000100:00100000:1.0:1767318643.936231:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318643.936958:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984965248:0@lo:400:kworker.0 00000100:00100000:1.0:1767318643.939240:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984965888 00000100:00100000:1.0:1767318643.939244:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984965888 00000100:00100000:1.0:1767318643.945259:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2746 00000100:00100000:1.0:1767318643.945267:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984965888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318643.945302:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984965888:12345-0@lo:400:kworker.0 Request processed in 36us (7863us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318643.945308:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2746 00010000:00010000:1.0:1767318648.667328:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318649.001613:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318649.005320:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920516096 00000100:00100000:1.0:1767318649.005333:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920516096 00000100:00100000:1.0:1767318649.005381:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4457 00000100:00100000:1.0:1767318649.005387:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920516096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318649.005410:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318649.005480:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920516096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 91us (269us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318649.005485:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4457 00000100:00080000:1.0:1767318649.051551:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318649.051561:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318649.051565:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318649.051569:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318649.051573:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318649.051583:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3892 00000100:00080000:1.0:1767318649.051588:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318649.051595:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318649.051642:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61c00 x1853163984966272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318649.052610:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318649.052616:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318649.052618:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318649.052623:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3892 00000100:00080000:1.0:1767318649.052626:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318649.052630:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318649.052650:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163984966400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318649.053939:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318649.053946:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318649.053949:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318649.053954:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3892 00000100:00080000:1.0:1767318649.053958:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318649.053964:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318649.053989:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62300 x1853163984966656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318649.055529:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318649.055534:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318649.055537:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318649.055542:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3892 00000100:00080000:1.0:1767318649.055545:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318649.055549:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318649.055571:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61880 x1853163984966912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318649.055641:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984967040:0@lo:400:kworker.0 00000100:00100000:1.0:1767318649.055673:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318649.055866:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318649.055883:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984967168:0@lo:400:kworker.0 00000100:00100000:1.0:1767318649.055987:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318649.056021:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984967168 00000100:00100000:1.0:1767318649.056026:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984967168 00000100:00100000:1.0:1767318649.057016:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2748 00000100:00100000:1.0:1767318649.057026:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984967168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318649.057089:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984967168:12345-0@lo:400:kworker.0 Request processed in 66us (1106us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318649.057096:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2748 00000100:00100000:1.0:1767318649.563570:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984967424:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318649.563600:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318649.563644:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984967424 00000100:00100000:1.0:1767318649.563648:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984967424 00000100:00100000:1.0:1767318649.563682:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1896 00000100:00100000:1.0:1767318649.563701:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984967424:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318649.563730:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318649.563775:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984967424:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318649.563786:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1896 00000100:00100000:1.0:1767318649.563802:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1897 00000100:00100000:1.0:1767318649.563808:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090000000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984967296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318649.563822:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318649.571356:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090000000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984967296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 7536us (7842us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318649.571374:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1897 00000100:00100000:1.0:1767318649.571479:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984967424:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318654.122923:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920516352 00000100:00100000:1.0:1767318654.122932:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920516352 00000100:00100000:1.0:1767318654.122977:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920516480 00000100:00100000:1.0:1767318654.122980:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920516480 00000100:00100000:1.0:1767318654.123017:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4144 00000100:00100000:1.0:1767318654.123028:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920516480:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318654.123094:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920516480:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 68us (443us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318654.123103:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4144 00000100:00100000:1.0:1767318654.123120:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4461 00000100:00100000:1.0:1767318654.123125:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920516352:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318654.123145:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318654.123208:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920516352:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 80us (614us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318654.123216:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4461 00000100:00080000:1.0:1767318654.173298:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318654.173308:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318654.173312:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318654.173325:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318654.173329:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318654.173337:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3897 00000100:00080000:1.0:1767318654.173343:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318654.173349:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318654.173389:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163984967552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318654.173414:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318654.173418:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318654.173420:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318654.173424:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3897 00000100:00080000:1.0:1767318654.173426:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318654.173429:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318654.173446:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61880 x1853163984967680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318654.173482:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318654.173486:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318654.173488:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318654.173491:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3897 00000100:00080000:1.0:1767318654.173493:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318654.173496:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318654.173507:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63b80 x1853163984967936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318654.173522:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318654.173525:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318654.173526:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318654.173530:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3897 00000100:00080000:1.0:1767318654.173532:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318654.173535:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318654.173544:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163984968192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318654.173588:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984968448:0@lo:400:kworker.0 00000100:00100000:1.0:1767318654.173648:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318654.173714:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318654.173720:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984968320:0@lo:400:kworker.0 00000100:00100000:1.0:1767318654.173744:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318654.173772:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984968448 00000100:00100000:1.0:1767318654.173779:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984968448 00000100:00100000:1.0:1767318654.173816:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984968320 00000100:00100000:1.0:1767318654.173818:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984968320 00000100:00100000:1.0:1767318654.173833:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2749 00000100:00100000:1.0:1767318654.173839:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984968448:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318654.173879:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984968448:12345-0@lo:400:kworker.0 Request processed in 40us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318654.173885:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2749 00000100:00100000:1.0:1767318654.173897:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2750 00000100:00100000:1.0:1767318654.173902:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984968320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318654.173920:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984968320:12345-0@lo:400:kworker.0 Request processed in 18us (176us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318654.173926:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2750 00000100:00100000:1.0:1767318654.173952:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984968320:0@lo:400:kworker.0 00000100:00100000:1.0:1767318654.173960:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984968448:0@lo:400:kworker.0 00000100:00100000:1.0:1767318654.683637:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984968704 00000100:00100000:1.0:1767318654.683646:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984968704 00000100:00100000:1.0:1767318654.683693:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984968576 00000100:00100000:1.0:1767318654.683695:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984968576 00000100:00100000:1.0:1767318654.683715:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1898 00000100:00100000:1.0:1767318654.683723:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984968704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318654.683749:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318654.690481:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984968704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 6747us (6984us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318654.690505:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1898 00010000:00010000:1.0:1767318654.812871:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318659.242820:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920516736 00000100:00100000:1.0:1767318659.242832:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920516736 00000100:00100000:1.0:1767318659.244521:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4147 00000100:00100000:1.0:1767318659.244532:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920516736:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318659.245227:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920516736:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 687us (2491us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318659.245236:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4147 00000100:00080000:1.0:1767318659.292129:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318659.292141:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318659.292144:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318659.292149:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318659.292154:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318659.292178:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3902 00000100:00080000:1.0:1767318659.292184:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318659.292191:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318659.292241:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76680 x1853163984968832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318659.292268:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318659.292272:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318659.292274:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318659.292277:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3902 00000100:00080000:1.0:1767318659.292279:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318659.292283:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318659.292296:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77b80 x1853163984968960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318659.292333:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318659.292348:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318659.292350:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318659.292353:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3902 00000100:00080000:1.0:1767318659.292356:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318659.292359:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318659.292370:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163984969216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318659.292395:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318659.292399:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318659.293293:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318659.293298:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3902 00000100:00080000:1.0:1767318659.293301:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318659.293304:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318659.293319:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163984969472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318659.299177:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2751 00000100:00100000:1.0:1767318659.299203:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090000e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984969600:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318659.299332:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090000e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984969600:12345-0@lo:400:kworker.0 Request processed in 142us (4950us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318659.299350:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2751 00000100:00100000:1.0:1767318659.299376:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2752 00000100:00100000:1.0:1767318659.299384:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984969728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318659.299436:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984969728:12345-0@lo:400:kworker.0 Request processed in 53us (4640us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318659.299442:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2752 00000100:00100000:1.0:1767318659.299495:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984969728:0@lo:400:kworker.0 00000100:00100000:1.0:1767318659.803900:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984969856 00000100:00100000:1.0:1767318659.803908:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984969856 00000100:00100000:1.0:1767318659.803953:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984969984 00000100:00100000:1.0:1767318659.803956:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984969984 00000100:00100000:1.0:1767318659.803977:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1900 00000100:00100000:1.0:1767318659.803985:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984969856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318659.804027:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318659.804072:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984969856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (539us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318659.804080:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1900 00010000:00010000:1.0:1767318660.955636:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318663.350545:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920517120 00000100:00100000:1.0:1767318663.351119:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920517120 00000100:00100000:1.0:1767318663.351312:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920516992 00000100:00100000:1.0:1767318663.351317:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920516992 00000100:00080000:1.0:1767318663.392102:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.392108:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318663.392111:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.392115:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 3887 00000100:00080000:1.0:1767318663.392119:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.392125:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318663.392196:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75500 x1853163984970112/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.392213:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.392216:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.392218:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.392222:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.392224:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.392227:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318663.392243:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163984970240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.392251:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.392254:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.392256:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.392259:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.392261:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.392264:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318663.392275:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163984970368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.392310:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.392314:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.392316:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.392319:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.392321:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.392324:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318663.392335:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163984970624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.392360:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.392363:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.392365:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.392368:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.392370:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.392373:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318663.392384:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76680 x1853163984970880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318663.392461:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984971008:0@lo:400:kworker.0 00000100:00100000:1.0:1767318663.392521:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318663.392574:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318663.392580:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984970752:0@lo:400:kworker.0 00000100:00100000:1.0:1767318663.392606:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318663.392639:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984971008 00000100:00100000:1.0:1767318663.392646:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984971008 00000100:00100000:1.0:1767318663.392695:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2753 00000100:00100000:1.0:1767318663.392704:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984971008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318663.392753:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984971008:12345-0@lo:400:kworker.0 Request processed in 50us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318663.392760:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2753 00000100:00100000:1.0:1767318663.392806:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984971008:0@lo:400:kworker.0 00000100:00100000:1.0:1767318663.392826:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984970752 00000100:00100000:1.0:1767318663.392830:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984970752 00000100:00100000:1.0:1767318663.392858:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4151 00000100:00100000:1.0:1767318663.392864:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984970752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318663.392909:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984970752:12345-0@lo:400:kworker.0 Request processed in 45us (303us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318663.392915:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4151 00000100:00100000:1.0:1767318663.392943:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984970752:0@lo:400:kworker.0 00010000:00010000:1.0:1767318667.103358:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318668.508518:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce090003b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984971776:0@lo:400:kworker.0 00000100:00100000:1.0:1767318668.508549:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318668.508580:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318668.508585:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984972416:0@lo:400:kworker.0 00000100:00100000:1.0:1767318668.508608:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318668.508635:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984971776 00000100:00100000:1.0:1767318668.508639:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984971776 00000100:00100000:1.0:1767318668.508672:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4155 00000100:00100000:1.0:1767318668.508678:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163984971776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318668.508711:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163984971776:12345-0@lo:400:kworker.0 Request processed in 33us (161us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318668.508717:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4155 00000100:00080000:1.0:1767318668.508732:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318668.508748:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090003b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984971776:0@lo:400:kworker.0 00000100:00080000:1.0:1767318668.510571:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318668.510576:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318668.510601:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984972416:0@lo:400:kworker.0 00010000:00010000:1.0:1767318673.243661:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318673.578010:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920517504 00000100:00100000:1.0:1767318673.578024:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920517504 00000100:00100000:1.0:1767318673.578090:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4477 00000100:00100000:1.0:1767318673.578100:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920517504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318673.578132:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318673.578292:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920517504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 190us (404us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318673.578301:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4477 00000100:00100000:1.0:1767318673.579735:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920517632 00000100:00100000:1.0:1767318673.579747:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920517632 00000100:00100000:1.0:1767318673.579824:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4156 00000100:00100000:1.0:1767318673.579833:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920517632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318673.579899:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920517632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 68us (1620us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318673.579907:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4156 00000100:00100000:3.0:1767318673.629094:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984972928:0@lo:38:kworker.0 00000100:00080000:3.0:1767318673.629104:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318673.629107:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318673.629110:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984973184:0@lo:38:kworker.0 00000100:00080000:3.0:1767318673.629117:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318673.629120:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318673.629123:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984973440:0@lo:38:kworker.0 00000100:00100000:1.0:1767318673.630257:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984973056 00000100:00100000:1.0:1767318673.630261:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984973056 00000100:00100000:1.0:1767318673.630304:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4158 00000100:00100000:1.0:1767318673.630310:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163984973056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318673.630359:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163984973056:12345-0@lo:400:kworker.0 Request processed in 50us (506us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318673.630364:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4158 00000100:00100000:1.0:1767318675.164296:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984973824 00000100:00100000:1.0:1767318675.164306:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984973824 00000100:00100000:1.0:1767318675.164372:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984973952 00000100:00100000:1.0:1767318675.164374:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984973952 00000100:00100000:1.0:1767318675.164399:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1906 00000100:00100000:1.0:1767318675.164418:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984973824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318675.164454:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318675.164504:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984973824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (843us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318675.164511:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1906 00000100:00100000:1.0:1767318675.164529:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1907 00000100:00100000:1.0:1767318675.164534:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984973952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318675.164550:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318675.164576:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984973952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (614us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318675.164581:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1907 00000100:00100000:3.0:1767318678.703234:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920517888 00000100:00100000:3.0:1767318678.703249:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920517888 00000100:00100000:1.0:1767318678.703328:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920517760 00000100:00100000:1.0:1767318678.703335:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920517760 00000100:00100000:1.0:1767318678.703402:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4481 00000100:00100000:1.0:1767318678.703416:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920517760:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318678.703445:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318678.703482:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4159 00000100:00100000:3.0:1767318678.703494:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920517888:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318678.703539:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920517760:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 124us (1023us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318678.703548:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920517888:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 55us (1109us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318678.703555:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4481 00000100:00100000:3.0:1767318678.703563:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4159 00000100:00080000:3.0:1767318678.747437:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.747451:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318678.747455:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.747459:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.747465:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.747475:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.747481:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.747490:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318678.747550:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21180 x1853163984974080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318678.747580:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.747584:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.747587:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.747590:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.747594:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.747597:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318678.747613:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21880 x1853163984974208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318678.747729:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.747733:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.747735:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.747740:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.747742:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.747747:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318678.747762:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ce00 x1853163984974464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318678.747793:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.747797:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.747800:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.747807:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.747812:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.747815:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318678.747827:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ed80 x1853163984974720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318678.747895:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318678.747910:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984974976:0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.747955:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318678.748015:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984974976 00000100:00100000:1.0:1767318678.748026:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984974976 00000100:00100000:1.0:1767318678.748108:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2759 00000100:00100000:1.0:1767318678.748121:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984974976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.748207:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984974976:12345-0@lo:400:kworker.0 Request processed in 87us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318678.748216:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2759 00000100:00100000:1.0:1767318678.748246:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984974848 00000100:00100000:1.0:1767318678.748250:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984974848 00000100:00100000:1.0:1767318678.748283:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2760 00000100:00100000:1.0:1767318678.748291:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984974848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.748334:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984974848:12345-0@lo:400:kworker.0 Request processed in 44us (105us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318678.748341:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2760 00000100:00100000:1.0:1767318678.748385:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984974976:0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.748410:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984974464 00000100:00100000:1.0:1767318678.748415:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984974464 00000100:00100000:1.0:1767318678.748444:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984974592 00000100:00100000:1.0:1767318678.748449:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984974592 00000100:00100000:1.0:1767318678.748476:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4160 00000100:00100000:1.0:1767318678.748486:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163984974592:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.748538:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163984974592:12345-0@lo:400:kworker.0 Request processed in 52us (243us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318678.748551:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4160 00000100:00100000:1.0:1767318678.748574:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4484 00000100:00100000:1.0:1767318678.748583:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984974720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318678.748596:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318678.748634:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984974720:12345-0@lo:38:kworker.0 Request processed in 50us (496us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318678.748642:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4484 00000100:00100000:1.0:1767318678.748678:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163984974592:0@lo:400:kworker.0 00000100:00080000:1.0:1767318678.748722:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318678.748729:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318678.748732:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318678.748768:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318678.748775:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318678.748780:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984974720:0@lo:38:kworker.0 00000100:00080000:1.0:1767318678.748791:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318678.748794:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318678.748798:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984974080:0@lo:38:kworker.0 00000100:00080000:1.0:1767318678.748809:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318678.748812:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318678.748816:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984974208:0@lo:38:kworker.0 00000100:00080000:1.0:1767318678.748823:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318678.748825:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318678.748828:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984974464:0@lo:38:kworker.0 00000100:00080000:1.0:1767318678.758758:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318679.389298:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318680.283533:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984975104:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318680.283633:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318680.284428:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318680.285090:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318680.298711:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984975104:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318683.818702:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318683.824689:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920518144 00000100:00100000:3.0:1767318683.824699:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920518144 00000100:00100000:3.0:1767318683.824855:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4162 00000100:00100000:3.0:1767318683.824867:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920518144:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318683.824963:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920518144:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 98us (372us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318683.824971:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4162 00000100:00100000:1.0:1767318683.872162:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984976128:0@lo:400:kworker.0 00000100:00100000:1.0:1767318683.872213:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318683.872233:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318683.872239:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984975872:0@lo:400:kworker.0 00000100:00100000:1.0:1767318683.872259:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318683.872285:0:29513:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984976128 00000100:00100000:1.0:1767318683.872289:0:29513:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984976128 00000100:00100000:1.0:1767318683.883263:0:29513:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2762 00000100:00100000:1.0:1767318683.883290:0:29513:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984976128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318683.883399:0:29513:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984976128:12345-0@lo:400:kworker.0 Request processed in 124us (11184us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318683.883406:0:29513:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2762 00000100:00100000:3.0:1767318685.403356:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984976384:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318685.403428:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318685.403992:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318685.404370:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318685.420888:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984976384:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318685.534334:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767318688.944223:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920518272 offset 0 length 520: 4 00000100:00100000:1.0:1767318688.944269:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318688.945368:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318688.947234:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920518528 00000100:00100000:3.0:1767318688.947246:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920518528 00000100:00100000:3.0:1767318688.951922:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4165 00000100:00100000:3.0:1767318688.951940:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090002d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920518528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318688.952044:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090002d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920518528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (6678us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318688.952053:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4165 00000100:00080000:1.0:1767318688.952667:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318688.987893:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318688.987899:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318688.987902:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318688.987907:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 3911 00000100:00080000:1.0:1767318688.987915:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318688.987922:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318688.987959:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce090002a00 x1853163984976640/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318688.987973:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318688.987977:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318688.987978:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318688.987982:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3932 00000100:00080000:1.0:1767318688.987984:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318688.987987:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318688.988000:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce090000380 x1853163984976768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318688.988008:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318688.988011:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318688.988013:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318688.988016:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3932 00000100:00080000:1.0:1767318688.988018:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318688.988021:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318688.988032:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce090003b80 x1853163984976896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318688.988073:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318688.988076:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318688.988078:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318688.988081:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3932 00000100:00080000:1.0:1767318688.988083:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318688.988085:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318688.988096:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163984977152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318688.988120:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318688.988123:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318688.988125:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318688.988128:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3932 00000100:00080000:1.0:1767318688.988131:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318688.988134:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318688.988143:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76680 x1853163984977408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318688.988194:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984977664:0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.988254:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318688.988443:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318688.988449:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163984977024:0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.988475:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318688.988504:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984977280:0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.988526:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318688.988541:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318688.988545:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984977536:0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.988567:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318688.988597:0:29513:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984977664 00000100:00100000:1.0:1767318688.988606:0:29513:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984977664 00000100:00100000:1.0:1767318688.988664:0:29513:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984977536 00000100:00100000:1.0:1767318688.988667:0:29513:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984977536 00000100:00100000:1.0:1767318688.988689:0:29513:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2763 00000100:00100000:1.0:1767318688.988696:0:29513:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984977664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.988741:0:29513:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984977664:12345-0@lo:400:kworker.0 Request processed in 45us (486us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318688.988748:0:29513:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2763 00000100:00100000:1.0:1767318688.988763:0:29513:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2764 00000100:00100000:1.0:1767318688.988769:0:29513:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984977536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.988796:0:29513:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984977536:12345-0@lo:400:kworker.0 Request processed in 27us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318688.988802:0:29513:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2764 00000100:00080000:1.0:1767318688.988825:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318688.988829:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318688.988853:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984977536:0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.988875:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984977664:0@lo:400:kworker.0 00000100:00100000:3.0:1767318688.990009:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984976896 00000100:00100000:3.0:1767318688.990014:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984976896 00000100:00100000:3.0:1767318688.990042:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1613 00000100:00100000:3.0:1767318688.990047:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984976896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318688.990058:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318688.990090:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984976896:12345-0@lo:38:kworker.0 Request processed in 42us (642us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318688.990100:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1613 00000100:00080000:3.0:1767318688.990122:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318688.990126:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318688.990129:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090003b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984976896:0@lo:38:kworker.0 00000100:00080000:1.0:1767318688.991403:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318690.525139:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318690.525150:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984977920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318690.525217:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318690.525651:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984977792 00000100:00100000:3.0:1767318690.525661:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984977792 00000100:00100000:3.0:1767318690.525728:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984977920 00000100:00100000:3.0:1767318690.525731:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984977920 00000100:00100000:3.0:1767318690.525752:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1912 00000100:00100000:3.0:1767318690.525761:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984977792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318690.525790:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318690.525839:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984977792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (965us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318690.525847:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1912 00000100:00100000:3.0:1767318690.525864:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1913 00000100:00100000:3.0:1767318690.525870:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984977920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318690.525890:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318690.525917:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984977920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (700us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318690.525923:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1913 00000100:00080000:1.0:1767318690.527214:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318690.527221:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318690.527265:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984977920:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318691.676548:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318694.061535:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318694.108011:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318694.108032:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318694.108035:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318694.108040:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318694.108044:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318694.108052:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3937 00000100:00080000:3.0:1767318694.108057:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318694.108063:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318694.108106:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73df80 x1853163984978048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318694.109026:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318694.109030:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318694.109032:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318694.109036:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3937 00000100:00080000:3.0:1767318694.109039:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318694.109044:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318694.109062:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163984978176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318694.112377:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984978176 00000100:00100000:1.0:1767318694.112386:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984978176 00000100:00100000:1.0:1767318694.112428:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1614 00000100:00100000:1.0:1767318694.112435:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090003b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984978176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318694.112455:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318694.112505:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090003b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984978176:12345-0@lo:38:kworker.0 Request processed in 60us (241us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318694.112512:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1614 00000100:00080000:3.0:1767318694.121796:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318694.121801:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318694.121805:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318694.121810:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3937 00000100:00080000:3.0:1767318694.121815:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318694.121821:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318694.121939:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163984978304:0@lo:400:kworker.0 00000100:00100000:1.0:1767318694.122010:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318694.122055:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984978304 00000100:00100000:1.0:1767318694.122061:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984978304 00000100:00100000:1.0:1767318694.122109:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4169 00000100:00100000:1.0:1767318694.122117:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163984978304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318694.122179:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163984978304:12345-0@lo:400:kworker.0 Request processed in 47us (154us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318694.122186:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4169 00000100:00080000:1.0:1767318694.122203:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318694.122236:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163984978304:0@lo:400:kworker.0 00000100:00100000:3.0:1767318694.122967:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c380 x1853163984978432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318694.123014:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318694.123018:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318694.123020:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318694.123025:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3937 00000100:00080000:3.0:1767318694.123028:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318694.123031:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318694.123044:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f480 x1853163984978688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318694.123358:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984978944 00000100:00100000:1.0:1767318694.123363:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984978944 00000100:00100000:1.0:1767318694.123407:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984978816 00000100:00100000:1.0:1767318694.123411:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984978816 00000100:00100000:1.0:1767318694.123430:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2765 00000100:00100000:1.0:1767318694.123437:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984978944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318694.123473:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984978944:12345-0@lo:400:kworker.0 Request processed in 36us (312us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318694.123479:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2765 00000100:00100000:1.0:1767318694.123490:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2766 00000100:00100000:1.0:1767318694.123496:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984978816:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318694.123522:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984978816:12345-0@lo:400:kworker.0 Request processed in 27us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318694.123529:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2766 00000100:00100000:1.0:1767318694.123561:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984978560 00000100:00100000:1.0:1767318694.123565:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984978560 00000100:00100000:1.0:1767318694.123589:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4170 00000100:00100000:1.0:1767318694.123594:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163984978560:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318694.123631:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163984978560:12345-0@lo:400:kworker.0 Request processed in 37us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318694.123637:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4170 00000100:00100000:1.0:1767318695.643569:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318695.643581:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984979200:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318695.646351:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318695.650099:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318695.653713:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984979072:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318695.658301:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984979200:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318697.819522:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318699.179947:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318699.180013:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:3.0:1767318699.180833:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318699.227699:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318699.227713:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318699.227716:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318699.227721:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318699.227726:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318699.227737:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3942 00000100:00080000:3.0:1767318699.227845:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318699.227856:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318699.227910:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ce00 x1853163984979328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318699.227940:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318699.227944:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318699.227946:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318699.227950:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3942 00000100:00080000:3.0:1767318699.227953:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318699.227956:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318699.227971:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ed80 x1853163984979456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318699.228014:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318699.228019:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318699.228021:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318699.228025:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3942 00000100:00080000:3.0:1767318699.228028:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318699.228032:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318699.228045:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73d180 x1853163984979712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318699.228073:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318699.228077:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318699.228079:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318699.228083:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3942 00000100:00080000:3.0:1767318699.228086:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318699.228089:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318699.228104:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f480 x1853163984979968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318699.228156:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984979840:0@lo:400:kworker.0 00000100:00100000:3.0:1767318699.228273:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318699.228878:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318699.228886:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984980096:0@lo:400:kworker.0 00000100:00100000:3.0:1767318699.228925:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318699.228964:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984980096 00000100:00100000:3.0:1767318699.228968:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984980096 00000100:00100000:3.0:1767318699.229022:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2767 00000100:00100000:3.0:1767318699.229032:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984980096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318699.229065:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984980096:12345-0@lo:400:kworker.0 Request processed in 36us (140us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318699.229071:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2767 00000100:00100000:3.0:1767318699.229083:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984980224 00000100:00100000:3.0:1767318699.229086:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984980224 00000100:00100000:3.0:1767318699.229107:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2768 00000100:00100000:3.0:1767318699.229112:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984980224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318699.229143:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984980224:12345-0@lo:400:kworker.0 Request processed in 31us (79us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318699.229148:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2768 00000100:00100000:3.0:1767318699.229220:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984980096:0@lo:400:kworker.0 00000100:00100000:3.0:1767318699.229231:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984979840:0@lo:400:kworker.0 00000100:00100000:1.0:1767318699.229823:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984979328:0@lo:38:kworker.0 00000100:00100000:1.0:1767318699.229898:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318699.229947:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984979456:0@lo:38:kworker.0 00000100:00100000:1.0:1767318699.229974:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318699.229997:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984979712:0@lo:38:kworker.0 00000100:00100000:1.0:1767318699.230016:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318699.230031:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984979968:0@lo:38:kworker.0 00000100:00100000:1.0:1767318699.230050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318699.230091:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984979328 00000100:00100000:1.0:1767318699.230099:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984979328 00000100:00100000:3.0:1767318699.233276:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984979584:0@lo:400:kworker.0 00000100:00100000:3.0:1767318699.233946:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984979712 00000100:00100000:3.0:1767318699.233958:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984979712 00000100:00100000:1.0:1767318699.244566:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4499 00000100:00100000:1.0:1767318699.244581:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984979328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318699.244611:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318699.244612:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4500 00000100:00100000:3.0:1767318699.244627:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984979712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318699.244649:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318699.245365:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984979328:12345-0@lo:38:kworker.0 Request processed in 781us (15462us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318699.245376:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4499 00000100:00100000:3.0:1767318699.245428:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984979712:12345-0@lo:38:kworker.0 Request processed in 799us (15407us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318699.245441:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4500 00010000:00010000:1.0:1767318699.867985:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318700.763407:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318700.763436:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984980480:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318700.763505:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318700.763738:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984980480 00000100:00100000:1.0:1767318700.763746:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984980480 00000100:00100000:1.0:1767318700.763786:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984980352 00000100:00100000:1.0:1767318700.763789:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984980352 00000100:00100000:1.0:1767318700.763809:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1916 00000100:00100000:1.0:1767318700.763816:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984980480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318700.763847:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318700.763890:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984980480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (391us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318700.763898:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1916 00000100:00100000:1.0:1767318700.763913:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1917 00000100:00100000:1.0:1767318700.763920:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984980352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318700.763933:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318700.766973:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984980352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3043us (3296us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318700.766990:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1917 00000100:00100000:1.0:1767318700.767066:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984980480:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318703.963591:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318704.299114:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4501 00000100:00100000:1.0:1767318704.299120:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920519168:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318704.299138:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318704.299192:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920519168:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (314us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318704.299198:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4501 00000100:00080000:3.0:1767318704.347519:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.347538:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318704.347540:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.347545:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.347548:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.347556:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.347560:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.347566:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318704.347603:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ca80 x1853163984980608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318704.347627:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.347630:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.347632:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.347635:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.347638:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.347641:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318704.347664:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c380 x1853163984980736/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318704.347706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984980608:0@lo:38:kworker.0 00000100:00080000:3.0:1767318704.347770:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.347774:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.347775:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.347779:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.347781:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.347785:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318704.347798:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e300 x1853163984980992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318704.347824:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.347827:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.347829:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.347834:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.347837:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.347840:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318704.347851:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73d180 x1853163984981248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318704.347897:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984981504:0@lo:400:kworker.0 00000100:00100000:1.0:1767318704.355647:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318704.355672:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984980736:0@lo:38:kworker.0 00000100:00100000:1.0:1767318704.355698:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318704.355724:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984980992:0@lo:38:kworker.0 00000100:00100000:1.0:1767318704.355744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318704.355759:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984981248:0@lo:38:kworker.0 00000100:00100000:1.0:1767318704.355778:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318704.355799:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318704.355804:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318704.355808:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984980608:0@lo:38:kworker.0 00000100:00100000:1.0:1767318704.355830:0:29513:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984981376 00000100:00100000:1.0:1767318704.355833:0:29513:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984981376 00000100:00100000:1.0:1767318704.355864:0:29513:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2769 00000100:00100000:1.0:1767318704.355870:0:29513:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984981376:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318704.355900:0:29513:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984981376:12345-0@lo:400:kworker.0 Request processed in 30us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318704.355905:0:29513:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2769 00000100:00100000:1.0:1767318704.355931:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984981376:0@lo:400:kworker.0 00000100:00080000:1.0:1767318704.355948:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318704.355951:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318704.355954:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984980992:0@lo:38:kworker.0 00000100:00080000:1.0:1767318704.355961:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318704.355963:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318704.355966:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984981248:0@lo:38:kworker.0 00000100:00080000:1.0:1767318704.356101:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318704.356106:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318704.356108:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984980736:0@lo:38:kworker.0 00000100:00100000:3.0:1767318704.362601:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318704.366891:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984981504 00000100:00100000:1.0:1767318704.366908:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984981504 00000100:00100000:1.0:1767318704.366990:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2770 00000100:00100000:1.0:1767318704.366999:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984981504:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318704.367086:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984981504:12345-0@lo:400:kworker.0 Request processed in 88us (4491us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318704.367097:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2770 00000100:00080000:3.0:1767318704.367187:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318704.367226:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984981504:0@lo:400:kworker.0 00000100:00080000:3.0:1767318704.370820:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318704.371009:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163984981120:0@lo:400:kworker.0 00000100:00100000:1.0:1767318705.883623:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984981632 00000100:00100000:1.0:1767318705.883631:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984981632 00000100:00100000:1.0:1767318705.883685:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1918 00000100:00100000:1.0:1767318705.883695:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984981632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318705.883724:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318705.883769:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984981632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (233us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318705.883777:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1918 00000100:00100000:1.0:1767318705.884074:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984981760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318705.884101:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318705.884451:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984981760:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318706.014178:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318709.417427:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318709.417479:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318709.417700:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4505 00000100:00100000:1.0:1767318709.417708:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920519424:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318709.417729:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318709.417766:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920519424:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (282us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318709.417779:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4505 00000100:00080000:3.0:1767318709.418193:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318709.470084:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318709.470095:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318709.470099:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318709.470103:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318709.470108:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318709.470116:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3952 00000100:00080000:1.0:1767318709.470120:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318709.470126:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318709.470317:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163984981888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318709.470345:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318709.470348:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318709.470350:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318709.470353:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3952 00000100:00080000:1.0:1767318709.470356:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318709.470359:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318709.470373:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3100 x1853163984982016/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318709.470550:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984981888:0@lo:38:kworker.0 00000100:00100000:3.0:1767318709.470612:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318709.470673:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984982016:0@lo:38:kworker.0 00000100:00100000:3.0:1767318709.470697:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318709.470750:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984982016 00000100:00100000:3.0:1767318709.470757:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984982016 00000100:00100000:3.0:1767318709.470789:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1617 00000100:00100000:3.0:1767318709.470796:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984982016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318709.470815:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318709.470853:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984982016:12345-0@lo:38:kworker.0 Request processed in 57us (155us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318709.470860:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1617 00000100:00080000:3.0:1767318709.470909:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318709.470913:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318709.470916:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984982016:0@lo:38:kworker.0 00000100:00080000:1.0:1767318709.471092:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318709.471097:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318709.471099:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318709.471103:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3952 00000100:00080000:1.0:1767318709.471106:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318709.471109:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318709.471127:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163984982272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318709.471331:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984982272:0@lo:38:kworker.0 00000100:00100000:3.0:1767318709.471353:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318709.471387:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984981888 00000100:00100000:3.0:1767318709.471391:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984981888 00000100:00100000:3.0:1767318709.471407:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984982272 00000100:00100000:3.0:1767318709.471409:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984982272 00000100:00100000:3.0:1767318709.471421:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4506 00000100:00100000:3.0:1767318709.471426:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163984981888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318709.471436:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318709.471461:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163984981888:12345-0@lo:38:kworker.0 Request processed in 34us (849us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318709.471467:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4506 00000100:00100000:3.0:1767318709.471477:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4507 00000100:00100000:3.0:1767318709.471480:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163984982272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318709.471488:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318709.471505:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163984982272:12345-0@lo:38:kworker.0 Request processed in 24us (151us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318709.471510:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4507 00000100:00080000:3.0:1767318709.471527:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318709.471530:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318709.471533:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984982272:0@lo:38:kworker.0 00000100:00080000:3.0:1767318709.471541:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318709.471544:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318709.471547:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984981888:0@lo:38:kworker.0 00000100:00100000:3.0:1767318709.471569:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984982400:0@lo:400:kworker.0 00000100:00100000:3.0:1767318709.471591:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318709.471613:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984982400 00000100:00100000:3.0:1767318709.471616:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984982400 00000100:00100000:3.0:1767318709.471650:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4178 00000100:00100000:3.0:1767318709.471656:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984982400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318709.471693:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984982400:12345-0@lo:400:kworker.0 Request processed in 38us (101us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318709.471699:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4178 00000100:00080000:3.0:1767318709.471714:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318709.471732:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984982400:0@lo:400:kworker.0 00000100:00080000:1.0:1767318709.471808:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318709.471812:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318709.471814:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318709.471819:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3952 00000100:00080000:1.0:1767318709.471821:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318709.471824:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318709.471838:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77480 x1853163984982528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318709.477857:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984982784 00000100:00100000:1.0:1767318709.477865:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984982784 00000100:00100000:1.0:1767318709.477975:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2772 00000100:00100000:1.0:1767318709.477984:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984982784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318709.478041:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984982784:12345-0@lo:400:kworker.0 Request processed in 58us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318709.478048:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2772 00000100:00100000:1.0:1767318709.478101:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984982784:0@lo:400:kworker.0 00000100:00080000:3.0:1767318709.485764:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318710.107781:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318711.003665:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318711.003679:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984983040:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318711.003706:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318711.003754:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984982912 00000100:00100000:1.0:1767318711.003762:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984982912 00000100:00100000:1.0:1767318711.003813:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984983040 00000100:00100000:1.0:1767318711.003816:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984983040 00000100:00100000:1.0:1767318711.003835:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1920 00000100:00100000:1.0:1767318711.003843:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984982912:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318711.003870:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318711.019602:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984982912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 15749us (16044us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318711.019621:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1920 00010000:00010000:1.0:1767318712.155766:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318713.516729:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767318713.516763:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920519680 offset 0 length 520: 4 00000100:00100000:3.0:1767318713.516783:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318713.516817:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920519936 00000100:00100000:1.0:1767318713.516827:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920519936 00000100:00080000:3.0:1767318713.516881:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318713.522181:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4509 00000100:00100000:1.0:1767318713.522205:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920519808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318713.522230:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.522712:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920519808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 507us (5925us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.522722:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4509 00000100:00080000:1.0:1767318713.565269:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318713.565276:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318713.565280:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318713.565287:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 3937 00000100:00080000:1.0:1767318713.565291:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318713.565298:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318713.565330:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163984983168/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318713.565365:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318713.565369:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318713.565371:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318713.565375:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3957 00000100:00080000:1.0:1767318713.565377:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318713.565380:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318713.565395:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77480 x1853163984983296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318713.565404:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318713.565407:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318713.565409:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318713.565412:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3957 00000100:00080000:1.0:1767318713.565415:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318713.565418:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318713.565430:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75500 x1853163984983424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318713.565463:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318713.565467:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318713.565469:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318713.565472:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3957 00000100:00080000:1.0:1767318713.565475:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318713.565478:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318713.565491:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163984983680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318713.565510:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318713.565514:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318713.565515:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318713.565519:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3957 00000100:00080000:1.0:1767318713.565521:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318713.565523:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318713.565532:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163984983936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318713.565790:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984983552 00000100:00100000:3.0:1767318713.565798:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984983552 00000100:00100000:3.0:1767318713.565953:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4181 00000100:00100000:3.0:1767318713.565963:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163984983552:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318713.566122:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163984983552:12345-0@lo:400:kworker.0 Request processed in 161us (550us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318713.566131:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4181 00000100:00100000:3.0:1767318713.566272:0:29513:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984984192 00000100:00100000:3.0:1767318713.566277:0:29513:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984984192 00000100:00100000:1.0:1767318713.566678:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163984983552:0@lo:400:kworker.0 00000100:00100000:1.0:1767318713.566781:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163984983168:0@lo:250:kworker.0 00000400:00000100:1.0:1767318713.566817:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163984983168 offset 0 length 520: 4 00000100:00100000:1.0:1767318713.566840:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984983296:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.566861:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.566876:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984983424:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.566895:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.566909:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984983680:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.566927:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.572200:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984983936:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.572266:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.572335:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984983296 00000100:00100000:1.0:1767318713.572343:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984983296 00000100:00100000:1.0:1767318713.572377:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984983680 00000100:00100000:1.0:1767318713.572380:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984983680 00000100:00100000:1.0:1767318713.572397:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984983936 00000100:00100000:1.0:1767318713.572399:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984983936 00000100:00100000:1.0:1767318713.572424:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4510 00000100:00100000:1.0:1767318713.572430:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984983296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318713.572453:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.572495:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984983296:12345-0@lo:38:kworker.0 Request processed in 63us (5632us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.572502:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4510 00000100:00100000:1.0:1767318713.572518:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4511 00000100:00100000:1.0:1767318713.572523:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984983680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318713.572532:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.572554:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984983680:12345-0@lo:38:kworker.0 Request processed in 30us (5627us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.572559:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4511 00000100:00100000:1.0:1767318713.572569:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4512 00000100:00100000:1.0:1767318713.572572:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984983936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318713.572580:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.572600:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984983936:12345-0@lo:38:kworker.0 Request processed in 26us (335us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.572605:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4512 00000100:00080000:1.0:1767318713.572646:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.572652:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318713.572655:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984983296:0@lo:38:kworker.0 00000100:00080000:1.0:1767318713.572663:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.572666:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318713.572669:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984983680:0@lo:38:kworker.0 00000100:00080000:1.0:1767318713.572675:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.572678:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318713.572680:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984983936:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.572728:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984984064:0@lo:400:kworker.0 00000100:00100000:1.0:1767318713.572752:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.572777:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984984064 00000100:00100000:1.0:1767318713.572780:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984984064 00000100:00100000:3.0:1767318713.580736:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984984192:0@lo:400:kworker.0 00000100:00100000:1.0:1767318713.581529:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2774 00000100:00100000:1.0:1767318713.581540:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984984064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318713.581611:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984984064:12345-0@lo:400:kworker.0 Request processed in 74us (8857us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318713.581618:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2774 00000100:00100000:1.0:1767318713.581647:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984984064:0@lo:400:kworker.0 00000100:00100000:1.0:1767318716.123513:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1767318716.123537:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984984448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318716.123605:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318716.123650:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984984320:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318716.123671:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318716.123997:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318716.124003:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318716.251696:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318718.300284:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318718.638588:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318718.638663:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767318718.638871:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318718.644407:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4183 00000100:00100000:3.0:1767318718.644443:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920520192:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318718.644538:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920520192:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 116us (5872us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318718.644547:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4183 00000100:00100000:3.0:1767318718.644575:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4513 00000100:00100000:3.0:1767318718.644580:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920520064:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318718.644600:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318718.644622:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920520064:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (6041us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318718.644628:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4513 00000100:00100000:1.0:1767318718.684412:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984984576:0@lo:38:kworker.0 00000100:00100000:1.0:1767318718.684437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318718.684461:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984984704:0@lo:38:kworker.0 00000100:00100000:1.0:1767318718.684482:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318718.684499:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984984960:0@lo:38:kworker.0 00000100:00100000:1.0:1767318718.684517:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318718.684531:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984985216:0@lo:38:kworker.0 00000100:00100000:1.0:1767318718.684548:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318718.684580:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984984832 00000100:00100000:1.0:1767318718.684585:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984984832 00000100:00100000:1.0:1767318718.684612:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984985088 00000100:00100000:1.0:1767318718.684616:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984985088 00000100:00100000:1.0:1767318718.684626:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984984576 00000100:00100000:1.0:1767318718.684628:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984984576 00000100:00100000:1.0:1767318718.684636:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984984960 00000100:00100000:1.0:1767318718.684638:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984984960 00000100:00100000:1.0:1767318718.684644:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984985216 00000100:00100000:1.0:1767318718.684646:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984985216 00000100:00100000:1.0:1767318718.684655:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4184 00000100:00100000:1.0:1767318718.684660:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163984984832:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318718.684691:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163984984832:12345-0@lo:400:kworker.0 Request processed in 30us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318718.684696:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4184 00000100:00100000:1.0:1767318718.684708:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4185 00000100:00100000:1.0:1767318718.684713:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163984985088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318718.684736:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163984985088:12345-0@lo:400:kworker.0 Request processed in 23us (395us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318718.684742:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4185 00000100:00100000:1.0:1767318718.684753:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4514 00000100:00100000:1.0:1767318718.684757:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984984576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318718.684773:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318718.684801:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984984576:12345-0@lo:38:kworker.0 Request processed in 43us (363us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318718.684807:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4514 00000100:00100000:1.0:1767318718.684818:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4515 00000100:00100000:1.0:1767318718.684822:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984984960:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318718.684830:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318718.684854:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984984960:12345-0@lo:38:kworker.0 Request processed in 30us (336us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318718.684859:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4515 00000100:00100000:1.0:1767318718.684869:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4516 00000100:00100000:1.0:1767318718.684872:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984985216:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318718.684880:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318718.684901:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984985216:12345-0@lo:38:kworker.0 Request processed in 27us (352us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318718.684906:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4516 00000100:00080000:1.0:1767318718.684937:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318718.684942:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318718.684945:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984984576:0@lo:38:kworker.0 00000100:00080000:1.0:1767318718.684953:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318718.684956:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318718.684959:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984984960:0@lo:38:kworker.0 00000100:00080000:1.0:1767318718.684966:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318718.684968:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318718.684970:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984985216:0@lo:38:kworker.0 00000100:00080000:1.0:1767318718.685341:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318718.685345:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318718.685347:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984984704:0@lo:38:kworker.0 00000100:00100000:1.0:1767318721.246157:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984985600 00000100:00100000:1.0:1767318721.246190:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984985600 00000100:00100000:1.0:1767318721.246279:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1924 00000100:00100000:1.0:1767318721.246290:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984985600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318721.246327:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318721.246381:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984985600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (441us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318721.246389:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1924 00000100:00100000:3.0:1767318721.247115:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984985728 00000100:00100000:3.0:1767318721.247122:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984985728 00000100:00100000:3.0:1767318721.247200:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1925 00000100:00100000:3.0:1767318721.247210:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984985728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318721.247233:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318721.247295:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984985728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (452us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318721.247301:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1925 00010000:00010000:1.0:1767318722.398557:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318723.755979:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318723.764834:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:3.0:1767318723.803866:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318723.803881:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318723.803884:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318723.803888:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318723.803894:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318723.803904:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3966 00000100:00080000:3.0:1767318723.803909:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318723.803918:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318723.803985:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163984985856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318723.804047:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318723.804051:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318723.804052:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318723.804056:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3966 00000100:00080000:3.0:1767318723.804059:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318723.804063:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318723.804078:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3100 x1853163984985984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318723.804140:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318723.804144:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318723.804146:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318723.804150:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3966 00000100:00080000:3.0:1767318723.804153:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318723.804157:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318723.804204:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1180 x1853163984986240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318723.804226:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318723.804231:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318723.804232:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318723.804236:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3966 00000100:00080000:3.0:1767318723.804238:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318723.804242:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318723.804253:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3b80 x1853163984986496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318723.804315:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984986624:0@lo:400:kworker.0 00000100:00100000:3.0:1767318723.804362:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318723.804438:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984986624 00000100:00100000:1.0:1767318723.804448:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984986624 00000100:00100000:1.0:1767318723.804527:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2777 00000100:00100000:1.0:1767318723.804543:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984986624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318723.804608:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984986624:12345-0@lo:400:kworker.0 Request processed in 70us (245us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318723.804617:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2777 00000100:00100000:1.0:1767318723.804659:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4188 00000100:00100000:1.0:1767318723.804669:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984986368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318723.805321:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984986368:12345-0@lo:400:kworker.0 Request processed in 654us (884us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318723.805328:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4188 00000100:00080000:1.0:1767318723.805351:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318723.805356:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318723.805379:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984986368:0@lo:400:kworker.0 00000100:00100000:3.0:1767318723.805502:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984986624:0@lo:400:kworker.0 00000100:00100000:1.0:1767318723.805805:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984986752 00000100:00100000:1.0:1767318723.805812:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984986752 00000100:00100000:1.0:1767318723.805906:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2778 00000100:00100000:1.0:1767318723.805915:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984986752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318723.805977:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984986752:12345-0@lo:400:kworker.0 Request processed in 63us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318723.805983:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2778 00000100:00100000:3.0:1767318723.806240:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984986752:0@lo:400:kworker.0 00000100:00100000:1.0:1767318723.807571:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984985856 00000100:00100000:1.0:1767318723.807580:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984985856 00000100:00100000:1.0:1767318723.807632:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4519 00000100:00100000:1.0:1767318723.807641:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163984985856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318723.807667:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318723.812226:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984985984 00000100:00100000:3.0:1767318723.812245:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984985984 00000100:00100000:3.0:1767318723.812319:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1620 00000100:00100000:3.0:1767318723.812332:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984985984:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318723.812359:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318723.812473:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984985984:12345-0@lo:38:kworker.0 Request processed in 142us (4591us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318723.812484:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1620 00000100:00100000:1.0:1767318723.812606:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163984985856:12345-0@lo:38:kworker.0 Request processed in 4962us (5120us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318723.812615:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4519 00010000:00010000:1.0:1767318724.443529:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318726.364736:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984986880 00000100:00100000:1.0:1767318726.364748:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984986880 00000100:00100000:1.0:1767318726.364838:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1926 00000100:00100000:1.0:1767318726.364850:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984986880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318726.364888:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318726.364962:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984986880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 114us (809us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318726.364970:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1926 00000100:00100000:1.0:1767318726.368892:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987008 00000100:00100000:1.0:1767318726.368905:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987008 00000100:00100000:1.0:1767318726.369014:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1927 00000100:00100000:1.0:1767318726.369025:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984987008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318726.369093:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318726.369243:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984987008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 218us (616us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318726.369253:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1927 00010000:00010000:1.0:1767318728.541793:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318728.876120:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318728.924400:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984987648:0@lo:400:kworker.0 00000100:00100000:3.0:1767318728.924437:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318728.924475:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318728.924481:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984987392:0@lo:400:kworker.0 00000100:00100000:3.0:1767318728.924505:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318728.924538:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984988032 00000100:00100000:3.0:1767318728.924546:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984988032 00000100:00100000:3.0:1767318728.924623:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987904 00000100:00100000:3.0:1767318728.924627:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987904 00000100:00100000:3.0:1767318728.924655:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2779 00000100:00100000:3.0:1767318728.924665:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984988032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318728.924728:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984988032:12345-0@lo:400:kworker.0 Request processed in 63us (516us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318728.924736:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2779 00000100:00100000:3.0:1767318728.924756:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2780 00000100:00100000:3.0:1767318728.924762:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984987904:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318728.924789:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984987904:12345-0@lo:400:kworker.0 Request processed in 27us (471us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318728.924795:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2780 00000100:00080000:3.0:1767318728.924813:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318728.924820:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318728.924861:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984987904:0@lo:400:kworker.0 00000100:00100000:3.0:1767318728.924874:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984988032:0@lo:400:kworker.0 00000100:00100000:1.0:1767318728.925333:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987648 00000100:00100000:1.0:1767318728.925340:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987648 00000100:00100000:1.0:1767318728.925403:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987392 00000100:00100000:1.0:1767318728.925406:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987392 00000100:00100000:1.0:1767318728.925436:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4190 00000100:00100000:1.0:1767318728.925445:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984987648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318728.925498:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984987648:12345-0@lo:400:kworker.0 Request processed in 54us (1059us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318728.925504:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4190 00000100:00100000:1.0:1767318728.925518:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4191 00000100:00100000:1.0:1767318728.925525:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163984987392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318728.925553:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163984987392:12345-0@lo:400:kworker.0 Request processed in 29us (1047us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318728.925559:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4191 00000100:00100000:1.0:1767318728.925586:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984987136:0@lo:38:kworker.0 00000100:00100000:1.0:1767318728.925617:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318728.925645:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984987264:0@lo:38:kworker.0 00000100:00100000:1.0:1767318728.925671:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318728.925692:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984987520:0@lo:38:kworker.0 00000100:00100000:1.0:1767318728.925715:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318728.925731:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984987776:0@lo:38:kworker.0 00000100:00100000:1.0:1767318728.925752:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318728.925784:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987136 00000100:00100000:1.0:1767318728.925788:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987136 00000100:00100000:1.0:1767318728.925809:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987520 00000100:00100000:1.0:1767318728.925811:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987520 00000100:00100000:1.0:1767318728.925818:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987776 00000100:00100000:1.0:1767318728.925821:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987776 00000100:00100000:1.0:1767318728.925831:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4523 00000100:00100000:1.0:1767318728.925836:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984987136:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318728.925860:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318728.925911:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984987136:12345-0@lo:38:kworker.0 Request processed in 61us (281us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318728.925918:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4523 00000100:00100000:1.0:1767318728.925930:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4524 00000100:00100000:1.0:1767318728.925934:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984987520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318728.925944:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318728.925969:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984987520:12345-0@lo:38:kworker.0 Request processed in 33us (253us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318728.925975:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4524 00000100:00100000:1.0:1767318728.925986:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4525 00000100:00100000:1.0:1767318728.925990:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984987776:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318728.925999:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318728.926022:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984987776:12345-0@lo:38:kworker.0 Request processed in 30us (269us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318728.926028:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4525 00000100:00080000:1.0:1767318728.926057:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318728.926063:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318728.926067:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984987136:0@lo:38:kworker.0 00000100:00080000:1.0:1767318728.926080:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318728.926084:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318728.926087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984987520:0@lo:38:kworker.0 00000100:00080000:1.0:1767318728.926095:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318728.926098:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318728.926101:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984987776:0@lo:38:kworker.0 00000100:00100000:3.0:1767318728.927001:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984987264 00000100:00100000:3.0:1767318728.927013:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984987264 00000100:00100000:3.0:1767318728.927070:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1621 00000100:00100000:3.0:1767318728.927078:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984987264:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318728.927093:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318728.927159:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984987264:12345-0@lo:38:kworker.0 Request processed in 80us (1487us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318728.928103:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1621 00000100:00080000:1.0:1767318728.928319:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318728.928326:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318728.928330:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984987264:0@lo:38:kworker.0 00010000:00010000:1.0:1767318730.588702:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318731.485265:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984988160 00000100:00100000:1.0:1767318731.485829:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984988160 00000100:00080000:3.0:1767318731.486749:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318731.486826:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1929 00000100:00100000:1.0:1767318731.486837:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984988160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318731.486871:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318731.486939:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984988160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (2909us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318731.486946:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1929 00000100:00100000:3.0:1767318733.998833:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318733.998919:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767318733.999234:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318734.043853:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318734.043863:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318734.043867:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318734.043870:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318734.043875:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318734.043885:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318734.043890:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318734.043896:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318734.043933:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc54e00 x1853163984988416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318734.043962:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318734.043966:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318734.043968:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318734.043971:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318734.043973:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318734.043977:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318734.043989:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc57b80 x1853163984988544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318734.044026:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318734.044030:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318734.044032:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318734.044035:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318734.044037:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318734.044040:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318734.044050:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc54380 x1853163984988800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318734.044064:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318734.044067:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318734.044068:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318734.044073:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318734.044075:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318734.044079:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318734.044090:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc55c00 x1853163984989056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318734.044106:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163984988928:0@lo:400:kworker.0 00000100:00100000:1.0:1767318734.044142:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984989184:0@lo:400:kworker.0 00000100:00100000:1.0:1767318734.044565:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318734.044633:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318734.044642:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984989312:0@lo:400:kworker.0 00000100:00100000:1.0:1767318734.044667:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318734.044698:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984989184 00000100:00100000:1.0:1767318734.044703:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984989184 00000100:00100000:1.0:1767318734.044761:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984989312 00000100:00100000:1.0:1767318734.044766:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984989312 00000100:00100000:1.0:1767318734.044793:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2781 00000100:00100000:1.0:1767318734.044812:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984989184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318734.044858:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984989184:12345-0@lo:400:kworker.0 Request processed in 57us (294us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318734.044866:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2781 00000100:00100000:1.0:1767318734.044885:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2782 00000100:00100000:1.0:1767318734.044894:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984989312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318734.044929:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984989312:12345-0@lo:400:kworker.0 Request processed in 36us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318734.044937:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2782 00000100:00100000:3.0:1767318734.045376:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318734.045588:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984988544 00000100:00100000:1.0:1767318734.045595:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984988544 00000100:00100000:1.0:1767318734.045630:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1622 00000100:00100000:1.0:1767318734.045640:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984988544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318734.045656:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318734.045695:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984988544:12345-0@lo:38:kworker.0 Request processed in 55us (508us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318734.045707:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1622 00000100:00080000:1.0:1767318734.045871:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318734.046323:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163984988928:0@lo:400:kworker.0 00000100:00080000:1.0:1767318734.046349:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318734.046356:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318734.046360:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984988416:0@lo:38:kworker.0 00000100:00080000:1.0:1767318734.046527:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318734.046532:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318734.046535:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984988544:0@lo:38:kworker.0 00000100:00080000:1.0:1767318734.046544:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318734.046547:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318734.046549:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984988800:0@lo:38:kworker.0 00000100:00080000:1.0:1767318734.046556:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318734.046559:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318734.046562:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984989056:0@lo:38:kworker.0 00010000:00010000:1.0:1767318734.683852:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318736.603883:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318736.605419:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984989568:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318736.605484:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318736.605640:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984989568 00000100:00100000:3.0:1767318736.605648:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984989568 00000100:00100000:3.0:1767318736.605699:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1931 00000100:00100000:3.0:1767318736.605708:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb4450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984989568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318736.605735:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318736.605788:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb4450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984989568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (303us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318736.605796:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1931 00000100:00080000:3.0:1767318736.605815:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318736.605851:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984989568:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318736.731663:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318738.139766:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163984990080:0@lo:400:kworker.0 00000100:00100000:1.0:1767318738.139792:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318738.139836:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984990080 00000100:00100000:1.0:1767318738.139839:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984990080 00000100:00100000:3.0:1767318738.139849:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984989952 00000100:00100000:3.0:1767318738.139855:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984989952 00000100:00100000:1.0:1767318738.139879:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4530 00000100:00100000:1.0:1767318738.139884:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984989824:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318738.139892:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1623 00000100:00100000:3.0:1767318738.139899:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984989952:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318738.139905:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767318738.139912:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318738.141370:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984989952:12345-0@lo:38:kworker.0 Request processed in 1468us (1587us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318738.141378:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1623 00000100:00100000:1.0:1767318738.142282:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984989824:12345-0@lo:38:kworker.0 Request processed in 2393us (2572us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318738.142293:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4530 00000400:00000100:3.0:1767318739.116286:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920521216 offset 0 length 520: 4 00000100:00100000:3.0:1767318739.116351:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318739.116529:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920521344 00000100:00100000:1.0:1767318739.116546:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920521344 00000100:00100000:1.0:1767318739.116616:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4533 00000100:00100000:1.0:1767318739.116626:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920521344:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318739.116651:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318739.116752:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920521344:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 126us (401us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318739.116760:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4533 00000100:00100000:1.0:1767318739.124218:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00010000:00010000:1.0:1767318740.835057:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318741.726139:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984990976 00000100:00100000:3.0:1767318741.726145:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984990976 00000100:00100000:3.0:1767318741.726200:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1933 00000100:00100000:3.0:1767318741.726209:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984990976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318741.726229:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318741.734054:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984990976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 7837us (7973us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318741.734140:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1933 00010000:00010000:1.0:1767318742.876209:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318743.261434:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318743.261443:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318743.261446:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318743.261451:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318743.261454:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318743.261462:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3986 00000100:00080000:1.0:1767318743.261467:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318743.261485:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318743.261523:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76680 x1853163984991104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318743.261555:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318743.261558:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318743.261560:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318743.261563:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3986 00000100:00080000:1.0:1767318743.261565:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318743.261568:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318743.261580:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75500 x1853163984991232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318743.261625:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318743.261637:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318743.261638:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318743.261642:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3986 00000100:00080000:1.0:1767318743.261645:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318743.261648:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318743.261667:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163984991488/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318743.261682:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318743.261685:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318743.261687:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318743.261689:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3986 00000100:00080000:1.0:1767318743.261692:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318743.261695:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318743.261703:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f480 x1853163984991744/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318743.266767:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984991232 00000100:00100000:3.0:1767318743.266774:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984991232 00000100:00100000:3.0:1767318743.266815:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1624 00000100:00100000:3.0:1767318743.266820:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984991232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318743.266831:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318743.266864:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984991232:12345-0@lo:38:kworker.0 Request processed in 43us (3924us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318743.266870:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1624 00000100:00080000:3.0:1767318743.266895:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318743.266899:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318743.266901:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984991232:0@lo:38:kworker.0 00000100:00080000:3.0:1767318743.298216:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318743.298226:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318743.298231:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984991104:0@lo:38:kworker.0 00000100:00100000:1.0:1767318746.844018:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984992128:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318746.844092:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318746.844317:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984992256:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318746.844347:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318746.844452:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984992128 00000100:00100000:1.0:1767318746.844460:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984992128 00000100:00100000:1.0:1767318746.844611:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1935 00000100:00100000:1.0:1767318746.844625:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984992128:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318746.845085:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318746.845127:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984992128:12345-0@lo:13:osp-pre-1-1.0 Request processed in 506us (1034us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318746.845135:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1935 00000100:00100000:3.0:1767318746.845853:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984992256:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318746.971339:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318748.379277:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318748.379287:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318748.379290:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318748.379295:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318748.379299:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318748.379307:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3991 00000100:00080000:1.0:1767318748.379311:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318748.379318:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318748.379356:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524df80 x1853163984992384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318748.379373:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318748.379376:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318748.379378:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318748.379382:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3991 00000100:00080000:1.0:1767318748.379384:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767318748.379385:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984992384:0@lo:38:kworker.0 00000100:00080000:1.0:1767318748.379387:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318748.379400:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524fb80 x1853163984992512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318748.379438:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318748.379442:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318748.379444:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318748.379447:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3991 00000100:00080000:1.0:1767318748.379449:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318748.379452:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318748.379459:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318748.379465:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c380 x1853163984992768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318748.379479:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318748.379482:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318748.379483:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318748.379486:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3991 00000100:00080000:1.0:1767318748.379488:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318748.379491:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318748.379499:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f100 x1853163984993024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318748.379516:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984992512:0@lo:38:kworker.0 00000100:00100000:3.0:1767318748.379549:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318748.379573:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984992768:0@lo:38:kworker.0 00000100:00100000:3.0:1767318748.379593:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318748.379772:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984992384 00000100:00100000:1.0:1767318748.379776:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984992384 00000100:00100000:3.0:1767318748.379781:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984993024:0@lo:38:kworker.0 00000100:00100000:1.0:1767318748.379798:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984992768 00000100:00100000:1.0:1767318748.379802:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984992768 00000100:00100000:3.0:1767318748.379804:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318748.379836:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984993280:0@lo:400:kworker.0 00000100:00100000:1.0:1767318748.379917:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984993024 00000100:00100000:1.0:1767318748.379922:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984993024 00000100:00100000:1.0:1767318748.380134:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984992896 00000100:00100000:1.0:1767318748.380138:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984992896 00000100:00100000:1.0:1767318748.380262:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4201 00000100:00100000:1.0:1767318748.380270:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984992896:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318748.391102:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318748.391108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318748.391146:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984992896:12345-0@lo:400:kworker.0 Request processed in 10875us (11277us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318748.391156:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4201 00000100:00100000:3.0:1767318748.391184:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318748.391197:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984992640 00000100:00100000:1.0:1767318748.391202:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984992640 00000100:00080000:1.0:1767318748.391223:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318748.391241:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4538 00000100:00100000:1.0:1767318748.391256:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4202 00000100:00100000:3.0:1767318748.391258:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984992384:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318748.391267:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163984992640:12345-0@lo:400:kworker.0 00010000:00020000:3.0:1767318748.391283:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318748.391317:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163984992640:12345-0@lo:400:kworker.0 Request processed in 52us (1015us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318748.391324:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4202 00000100:00100000:3.0:1767318748.391339:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984992384:12345-0@lo:38:kworker.0 Request processed in 88us (11877us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318748.391343:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4539 00000100:00100000:1.0:1767318748.391352:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984992768:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318748.391355:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4538 00010000:00020000:1.0:1767318748.391368:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318748.391378:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4540 00000100:00100000:3.0:1767318748.391384:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984993024:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318748.391396:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318748.391400:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163984992768:12345-0@lo:38:kworker.0 Request processed in 44us (11802us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318748.391408:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4539 00000100:00100000:1.0:1767318748.391454:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984992640:0@lo:400:kworker.0 00000100:00100000:1.0:1767318748.391465:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984992896:0@lo:400:kworker.0 00000100:00100000:1.0:1767318748.392064:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984993280 00000100:00100000:1.0:1767318748.392070:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984993280 00000100:00100000:1.0:1767318748.393225:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2787 00000100:00100000:1.0:1767318748.393243:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984993152:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318748.393316:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984993152:12345-0@lo:400:kworker.0 Request processed in 81us (2437us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318748.393323:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2787 00000100:00080000:1.0:1767318748.395394:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318748.401700:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163984993024:12345-0@lo:38:kworker.0 Request processed in 10305us (21886us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318748.401714:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4540 00010000:00010000:1.0:1767318749.022811:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318749.354593:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920521984 00000100:00100000:3.0:1767318749.354603:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920521984 00000100:00100000:1.0:1767318749.355009:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920521856 00000100:00100000:1.0:1767318749.355014:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920521856 00000100:00100000:1.0:1767318749.355062:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4203 00000100:00100000:1.0:1767318749.355073:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920521984:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318749.355192:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920521984:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 121us (800us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318749.355200:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4203 00000100:00100000:1.0:1767318749.355219:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4541 00000100:00100000:1.0:1767318749.355223:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920521856:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318749.355243:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318749.355281:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920521856:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (787us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318749.355286:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4541 00000100:00100000:3.0:1767318751.964896:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984993408 00000100:00100000:3.0:1767318751.964908:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984993408 00000100:00100000:3.0:1767318751.964979:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1936 00000100:00100000:3.0:1767318751.964993:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984993408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318751.965024:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767318751.965247:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318751.965252:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318751.981875:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984993408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 16875us (17996us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318751.981891:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1936 00010000:00010000:1.0:1767318753.115365:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318753.500726:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318753.500733:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318753.500752:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984993920 00000100:00100000:1.0:1767318753.500757:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984993920 00000100:00100000:1.0:1767318753.500810:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984994176 00000100:00100000:1.0:1767318753.500814:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984994176 00000100:00100000:1.0:1767318753.500835:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4204 00000100:00100000:1.0:1767318753.500843:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163984993920:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318753.500883:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163984993920:12345-0@lo:400:kworker.0 Request processed in 42us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318753.500890:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4204 00000100:00100000:1.0:1767318753.500902:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4205 00000100:00100000:1.0:1767318753.500908:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984994176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318753.500933:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984994176:12345-0@lo:400:kworker.0 Request processed in 25us (246us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318753.500939:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4205 00000100:00100000:1.0:1767318753.500967:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984994176:0@lo:400:kworker.0 00000100:00100000:1.0:1767318753.500976:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163984993920:0@lo:400:kworker.0 00000100:00100000:3.0:1767318753.503569:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984993664:0@lo:38:kworker.0 00000100:00100000:3.0:1767318753.503642:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318753.503704:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984993792:0@lo:38:kworker.0 00000100:00100000:3.0:1767318753.503730:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318753.503764:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984994048:0@lo:38:kworker.0 00000100:00100000:3.0:1767318753.503783:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318753.503811:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984994304:0@lo:38:kworker.0 00000100:00100000:3.0:1767318753.503829:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318753.503870:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984993664 00000100:00100000:3.0:1767318753.503877:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984993664 00000100:00100000:3.0:1767318753.503908:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984994048 00000100:00100000:3.0:1767318753.503911:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984994048 00000100:00100000:3.0:1767318753.503927:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984994304 00000100:00100000:3.0:1767318753.503929:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984994304 00000100:00100000:3.0:1767318753.503943:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4542 00000100:00100000:3.0:1767318753.503949:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984993664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318753.503969:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318753.504011:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984993664:12345-0@lo:38:kworker.0 Request processed in 60us (368us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318753.504017:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4542 00000100:00100000:3.0:1767318753.504033:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4543 00000100:00100000:3.0:1767318753.504038:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984994048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318753.504046:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318753.504065:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984994048:12345-0@lo:38:kworker.0 Request processed in 27us (282us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318753.504070:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4543 00000100:00100000:3.0:1767318753.504079:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4544 00000100:00100000:3.0:1767318753.504082:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984994304:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318753.504090:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318753.504109:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163984994304:12345-0@lo:38:kworker.0 Request processed in 25us (279us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318753.504114:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4544 00000100:00080000:3.0:1767318753.504153:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318753.504157:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318753.504161:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984993664:0@lo:38:kworker.0 00000100:00080000:3.0:1767318753.504194:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318753.504198:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318753.504201:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984994048:0@lo:38:kworker.0 00000100:00080000:3.0:1767318753.504207:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318753.504210:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318753.504212:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984994304:0@lo:38:kworker.0 00000100:00080000:3.0:1767318753.504578:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318753.504583:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318753.504586:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984993792:0@lo:38:kworker.0 00000100:00100000:1.0:1767318754.474274:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318754.474326:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767318754.474988:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318755.167312:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318757.083796:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984994688:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318757.083899:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318757.083965:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318757.083972:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984994816:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318757.083997:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318757.084043:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984994688 00000100:00100000:1.0:1767318757.084052:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984994688 00000100:00100000:1.0:1767318757.084125:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984994816 00000100:00100000:1.0:1767318757.084129:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984994816 00000100:00100000:1.0:1767318757.084161:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1938 00000100:00100000:1.0:1767318757.084189:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984994688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318757.084227:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318757.084275:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984994688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318757.084284:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1938 00000100:00100000:1.0:1767318757.084303:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1939 00000100:00100000:1.0:1767318757.084309:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984994816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318757.084325:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318757.084351:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163984994816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 42us (353us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318757.084357:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1939 00000100:00080000:1.0:1767318757.084383:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318757.084387:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318757.084440:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984994816:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318757.084455:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984994688:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318758.619786:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318758.619798:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318758.619801:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318758.619805:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318758.619809:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318758.619817:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4001 00000100:00080000:1.0:1767318758.619822:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318758.619829:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318758.619869:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163984994944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318758.619905:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318758.619909:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318758.619910:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318758.619914:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4001 00000100:00080000:1.0:1767318758.619917:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318758.619920:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318758.619933:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524e300 x1853163984995072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318758.619974:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318758.619979:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318758.619981:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318758.619984:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4001 00000100:00080000:1.0:1767318758.619987:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318758.619990:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318758.620002:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f100 x1853163984995328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318758.620018:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318758.620021:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318758.620023:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318758.620026:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4001 00000100:00080000:1.0:1767318758.620029:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318758.620032:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318758.620041:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163984995584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318758.620090:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984995712:0@lo:400:kworker.0 00000100:00100000:1.0:1767318758.620156:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318758.620242:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318758.620248:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984995840:0@lo:400:kworker.0 00000100:00100000:1.0:1767318758.620271:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318758.620525:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984995456:0@lo:400:kworker.0 00000100:00100000:3.0:1767318758.622993:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318758.623245:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984995840:0@lo:400:kworker.0 00000100:00100000:1.0:1767318758.623256:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163984995712:0@lo:400:kworker.0 00000100:00100000:1.0:1767318758.623294:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984995072 00000100:00100000:1.0:1767318758.623299:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984995072 00000100:00100000:3.0:1767318758.624706:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318758.624721:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163984995200:0@lo:400:kworker.0 00000100:00100000:3.0:1767318758.624764:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318758.624842:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1627 00000100:00100000:1.0:1767318758.624850:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984995072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318758.624866:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318758.625727:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163984995072:12345-0@lo:38:kworker.0 Request processed in 873us (3924us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318758.625736:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1627 00000100:00080000:1.0:1767318758.625789:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318758.625793:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318758.625797:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984995072:0@lo:38:kworker.0 00010000:00010000:1.0:1767318759.259512:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318759.596875:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920522496 00000100:00100000:1.0:1767318759.596887:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920522496 00000100:00100000:1.0:1767318759.596962:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920522368 00000100:00100000:1.0:1767318759.596966:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920522368 00000100:00100000:1.0:1767318759.596984:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4209 00000100:00100000:1.0:1767318759.596994:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920522496:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318759.597081:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920522496:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318759.597089:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4209 00000100:00100000:1.0:1767318759.597108:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4549 00000100:00100000:1.0:1767318759.597113:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920522368:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318759.597135:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318759.597161:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920522368:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (368us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318759.597218:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4549 00010000:00010000:1.0:1767318761.309210:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318762.203822:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984995968 00000100:00100000:1.0:1767318762.203831:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984995968 00000100:00100000:1.0:1767318762.203892:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1940 00000100:00100000:1.0:1767318762.203901:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984995968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318762.203929:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318762.203976:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163984995968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (234us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318762.203983:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1940 00000100:00100000:1.0:1767318762.204035:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984995968:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318762.206472:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163984996096:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318762.206543:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.695780:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767318763.695858:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920522624 offset 0 length 520: 4 00000100:00100000:1.0:1767318763.695885:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318763.697162:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920522880 00000100:00100000:3.0:1767318763.697341:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920522880 00000100:00100000:3.0:1767318763.697541:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920522752 00000100:00100000:3.0:1767318763.697545:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920522752 00000100:00100000:3.0:1767318763.697602:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4210 00000100:00100000:3.0:1767318763.697612:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920522880:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318763.697690:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920522880:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (1918us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318763.697697:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4210 00000100:00100000:3.0:1767318763.697716:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4550 00000100:00100000:3.0:1767318763.697720:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920522752:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318763.697741:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318763.697764:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920522752:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (1879us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318763.697770:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4550 00000100:00100000:1.0:1767318763.740791:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163984996480:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.740825:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.740859:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318763.740864:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984996736:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.740882:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.740926:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984996992 00000100:00100000:1.0:1767318763.740933:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984996992 00000100:00100000:1.0:1767318763.740968:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984997120 00000100:00100000:1.0:1767318763.740970:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984997120 00000100:00100000:1.0:1767318763.740986:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2793 00000100:00100000:1.0:1767318763.740993:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984996992:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741038:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984996992:12345-0@lo:400:kworker.0 Request processed in 45us (725us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318763.741043:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2793 00000100:00100000:1.0:1767318763.741056:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2794 00000100:00100000:1.0:1767318763.741061:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984997120:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741079:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984997120:12345-0@lo:400:kworker.0 Request processed in 19us (680us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318763.741085:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2794 00000100:00100000:1.0:1767318763.741114:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984997120:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741123:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984996992:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741149:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318738/real 1767318738] req@ffff8ce1adc98380 x1853163984989696/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318763 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318763.741188:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1adc98380 x1853163984989696/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318763 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318763.741199:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984996224:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.741221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.741256:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163984996352:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.741278:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.741298:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984996608:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.741316:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.741337:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984996480 00000100:00100000:1.0:1767318763.741340:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984996480 00000100:00100000:1.0:1767318763.741367:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984996736 00000100:00100000:1.0:1767318763.741370:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984996736 00000100:00100000:1.0:1767318763.741382:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984996224 00000100:00100000:1.0:1767318763.741384:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984996224 00000100:00100000:1.0:1767318763.741392:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984996608 00000100:00100000:1.0:1767318763.741394:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984996608 00000100:00100000:1.0:1767318763.741402:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4211 00000100:00100000:1.0:1767318763.741407:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e376680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163984996480:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741449:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e376680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163984996480:12345-0@lo:400:kworker.0 Request processed in 42us (623us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318763.741455:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4211 00000100:00100000:1.0:1767318763.741465:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4212 00000100:00100000:1.0:1767318763.741470:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984996736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741491:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163984996736:12345-0@lo:400:kworker.0 Request processed in 21us (608us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318763.741499:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4212 00000100:00100000:1.0:1767318763.741507:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4551 00000100:00100000:1.0:1767318763.741511:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984996224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318763.741527:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318763.741550:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984996224:12345-0@lo:38:kworker.0 Request processed in 37us (327us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318763.741555:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4551 00000100:00100000:1.0:1767318763.741577:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4552 00000100:00100000:1.0:1767318763.741581:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984996608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318763.741590:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318763.741608:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984996608:12345-0@lo:38:kworker.0 Request processed in 26us (292us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318763.741613:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4552 00000100:00100000:1.0:1767318763.741634:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163984996736:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741642:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163984996480:0@lo:400:kworker.0 00000100:00080000:1.0:1767318763.741660:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318763.741665:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318763.741668:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984996608:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.741674:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984996864:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.741692:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.741706:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984996864 00000100:00100000:1.0:1767318763.741709:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984996864 00000100:00100000:1.0:1767318763.741719:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4553 00000100:00100000:1.0:1767318763.741724:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984996864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318763.741732:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318763.741751:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984996864:12345-0@lo:38:kworker.0 Request processed in 25us (57us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318763.741755:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4553 00000100:00080000:1.0:1767318763.741771:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318763.741774:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318763.741777:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984996864:0@lo:38:kworker.0 00000100:00080000:1.0:1767318763.741782:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318763.741792:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318763.741795:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163984989696:0@lo:250:kworker.0 00000100:00080000:1.0:1767318763.741812:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318763.741816:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318763.741819:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318763.741822:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 3986 00000100:00080000:1.0:1767318763.741826:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318763.741829:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318763.741845:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adf100 x1853163984997248/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318763.741857:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318763.741861:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318763.741863:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318763.741865:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318763.741877:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318763.741881:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318763.741883:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318763.741886:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4011 00000100:00080000:1.0:1767318763.741888:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318763.741890:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318763.741901:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6addc00 x1853163984997504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318763.741913:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318763.741915:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318763.741917:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318763.741920:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4011 00000100:00080000:1.0:1767318763.741923:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318763.741927:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318763.741934:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6add180 x1853163984997760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318763.741965:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163984997632:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.741984:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.741999:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318763.742004:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ade300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163984997376:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.742018:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.742036:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6add500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984997888:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.742053:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.742068:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318763.742072:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6add880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163984998016:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.742086:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318763.742103:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984997888 00000100:00100000:1.0:1767318763.742105:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984997888 00000100:00100000:1.0:1767318763.742120:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984998016 00000100:00100000:1.0:1767318763.742122:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984998016 00000100:00100000:1.0:1767318763.742133:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2795 00000100:00100000:1.0:1767318763.742138:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984997888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.748025:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163984997888:12345-0@lo:400:kworker.0 Request processed in 24us (108us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318763.748050:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2795 00010000:00010000:1.0:1767318765.406634:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318767.323863:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984998272 00000100:00100000:1.0:1767318767.323882:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984998272 00000100:00100000:1.0:1767318767.324124:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1942 00000100:00100000:1.0:1767318767.324134:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984998272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318767.324191:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318767.324237:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163984998272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (487us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318767.324246:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1942 00010000:00010000:1.0:1767318767.456081:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318767.771343:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318767.771359:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318767.771363:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318767.771369:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318767.771375:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318767.771386:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4011 00000100:00080000:1.0:1767318767.771392:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318767.771401:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318767.771457:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6aded80 x1853163984998400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318767.771491:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318767.771495:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318767.771498:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318767.771501:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4011 00000100:00080000:1.0:1767318767.771504:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318767.771507:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318767.771523:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6addf80 x1853163984998528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318767.771594:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318767.771599:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318767.771600:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318767.771605:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4012 00000100:00080000:1.0:1767318767.771608:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318767.771611:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318767.771623:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ade300 x1853163984998784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318767.771638:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318767.771641:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318767.771643:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318767.771646:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4012 00000100:00080000:1.0:1767318767.771648:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318767.771651:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318767.771660:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6addc00 x1853163984999040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318767.772715:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6add180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163984998912:0@lo:400:kworker.0 00000100:00100000:3.0:1767318767.772726:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163984998656:0@lo:400:kworker.0 00000100:00100000:3.0:1767318767.772768:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6add500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984999168:0@lo:400:kworker.0 00000100:00100000:3.0:1767318767.772777:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6add880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163984999296:0@lo:400:kworker.0 00000100:00080000:3.0:1767318767.772800:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318767.772804:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318767.772807:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ade300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163984998784:0@lo:38:kworker.0 00000100:00100000:3.0:1767318767.772812:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6addc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984999040:0@lo:38:kworker.0 00000100:00100000:3.0:1767318767.772837:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318767.772857:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984999040 00000100:00100000:3.0:1767318767.772861:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984999040 00000100:00080000:1.0:1767318767.772900:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318767.773244:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4558 00000100:00100000:3.0:1767318767.773250:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984999040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318767.773266:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318767.773296:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984999040:12345-0@lo:38:kworker.0 Request processed in 44us (457us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318767.773302:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4558 00000100:00080000:3.0:1767318767.773396:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318767.773401:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318767.773405:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6addc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163984999040:0@lo:38:kworker.0 00000100:00080000:3.0:1767318767.773428:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318767.773431:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318767.773433:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6aded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163984998400:0@lo:38:kworker.0 00000100:00100000:1.0:1767318768.796695:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984999936:0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.796822:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985000192:0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.797812:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985000320:0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.797865:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318768.797929:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985000320 00000100:00100000:1.0:1767318768.797940:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985000320 00000100:00100000:1.0:1767318768.798304:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2799 00000100:00100000:1.0:1767318768.798314:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985000320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.798391:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985000320:12345-0@lo:400:kworker.0 Request processed in 78us (526us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318768.798400:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2799 00000100:00100000:1.0:1767318768.798454:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985000320:0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.802039:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318768.802090:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985000192 00000100:00100000:3.0:1767318768.802097:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985000192 00000100:00100000:3.0:1767318768.802143:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2800 00000100:00100000:3.0:1767318768.802177:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985000192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.802238:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985000192:12345-0@lo:400:kworker.0 Request processed in 87us (198us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318768.802248:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2800 00000100:00100000:3.0:1767318768.802317:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985000192:0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.802358:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984999552 00000100:00100000:3.0:1767318768.802362:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984999552 00000100:00100000:3.0:1767318768.802400:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1630 00000100:00100000:3.0:1767318768.802406:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984999552:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318768.802425:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318768.802454:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163984999552:12345-0@lo:38:kworker.0 Request processed in 47us (548us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318768.802460:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1630 00000100:00100000:1.0:1767318768.806379:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984999808 00000100:00100000:1.0:1767318768.806393:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984999808 00000100:00100000:1.0:1767318768.806440:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985000064 00000100:00100000:1.0:1767318768.806444:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985000064 00000100:00100000:1.0:1767318768.806461:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4559 00000100:00100000:1.0:1767318768.806470:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984999424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318768.806500:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318768.806732:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984999424:12345-0@lo:38:kworker.0 Request processed in 259us (9033us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318768.806741:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4559 00000100:00100000:1.0:1767318768.806763:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4560 00000100:00100000:1.0:1767318768.806767:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984999808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318768.806778:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318768.806805:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163984999808:12345-0@lo:38:kworker.0 Request processed in 36us (4826us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318768.806811:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4560 00000100:00100000:1.0:1767318768.806825:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4561 00000100:00100000:1.0:1767318768.806829:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985000064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318768.806839:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318768.806863:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985000064:12345-0@lo:38:kworker.0 Request processed in 32us (674us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318768.806869:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4561 00000100:00100000:3.0:1767318768.808433:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318768.808525:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163984999936 00000100:00100000:3.0:1767318768.808535:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163984999936 00000100:00100000:3.0:1767318768.808582:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4218 00000100:00100000:3.0:1767318768.808592:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984999936:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.808667:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163984999936:12345-0@lo:400:kworker.0 Request processed in 77us (241us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318768.808675:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4218 00000100:00100000:3.0:1767318768.808720:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163984999936:0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.809544:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163984999680:0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.813270:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318768.813321:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318768.815061:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4219 00000100:00100000:3.0:1767318768.815082:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ade300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920523136:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318768.815623:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ade300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920523136:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 548us (2298us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318768.815634:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4219 00000100:00100000:3.0:1767318768.815665:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4562 00000100:00100000:3.0:1767318768.815671:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6addc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920523008:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318768.815693:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318768.815723:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6addc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920523008:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (2466us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318768.815731:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4562 00010000:00010000:1.0:1767318771.547398:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318772.443836:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6addf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985000576:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318772.443910:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985000448 00000100:00100000:3.0:1767318772.443919:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985000448 00000100:00100000:3.0:1767318772.443989:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985000576 00000100:00100000:3.0:1767318772.443992:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985000576 00000100:00100000:3.0:1767318772.444013:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1944 00000100:00100000:3.0:1767318772.444027:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985000448:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318772.444055:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318772.448299:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985000448:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3472us (3616us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318772.448313:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1944 00000100:00100000:3.0:1767318772.449567:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6addf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985000576:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318773.595528:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318773.931790:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318773.931847:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318773.932033:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920523264 00000100:00100000:1.0:1767318773.932042:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920523264 00000100:00100000:1.0:1767318773.932094:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920523392 00000100:00100000:1.0:1767318773.932098:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920523392 00000100:00100000:1.0:1767318773.932157:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4220 00000100:00100000:1.0:1767318773.932201:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920523392:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318773.932287:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920523392:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 120us (437us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318773.932295:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4220 00000100:00100000:1.0:1767318773.932314:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4563 00000100:00100000:1.0:1767318773.932319:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920523264:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318773.932340:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318773.932366:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920523264:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (586us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318773.932373:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4563 00000100:00080000:1.0:1767318773.932565:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318773.979338:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318773.979349:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318773.979352:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318773.979356:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318773.979361:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318773.979369:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4017 00000100:00080000:1.0:1767318773.979373:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318773.979389:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318773.979428:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6add880 x1853163985000704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318773.979453:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318773.979457:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318773.979458:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318773.979462:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4017 00000100:00080000:1.0:1767318773.979464:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318773.979467:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318773.979480:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6add500 x1853163985000832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318773.979756:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985000704 00000100:00100000:3.0:1767318773.979765:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985000704 00000100:00100000:3.0:1767318773.979810:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4564 00000100:00100000:3.0:1767318773.979818:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985000704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318773.979841:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318773.979894:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985000704:12345-0@lo:38:kworker.0 Request processed in 74us (172us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318773.979903:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4564 00000100:00080000:1.0:1767318773.980485:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318773.980491:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318773.980493:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318773.980498:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4017 00000100:00080000:1.0:1767318773.980501:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318773.980505:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318773.980524:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6add180 x1853163985001088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318773.980719:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985001088 00000100:00100000:3.0:1767318773.980724:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985001088 00000100:00100000:3.0:1767318773.980754:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4565 00000100:00100000:3.0:1767318773.980760:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985001088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318773.980771:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318773.980817:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985001088:12345-0@lo:38:kworker.0 Request processed in 56us (185us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318773.980824:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4565 00000100:00100000:3.0:1767318773.984727:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985001216 00000100:00100000:3.0:1767318773.984735:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985001216 00000100:00100000:3.0:1767318773.984808:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985000960 00000100:00100000:3.0:1767318773.984812:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985000960 00000100:00100000:3.0:1767318773.984837:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4221 00000100:00100000:3.0:1767318773.984845:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985001216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318773.984899:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985001216:12345-0@lo:400:kworker.0 Request processed in 56us (566us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318773.984907:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4221 00000100:00100000:3.0:1767318773.984922:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4222 00000100:00100000:3.0:1767318773.984928:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985000960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318773.984952:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985000960:12345-0@lo:400:kworker.0 Request processed in 25us (554us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318773.984958:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4222 00000100:00080000:1.0:1767318773.985309:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318773.985314:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318773.985318:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318773.985323:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4017 00000100:00080000:1.0:1767318773.985327:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318773.985331:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318773.985360:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6add880 x1853163985001344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318773.985591:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318773.986465:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6addf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985001472:0@lo:400:kworker.0 00000100:00100000:3.0:1767318773.986495:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318777.565771:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985001856 00000100:00100000:1.0:1767318777.565778:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985001856 00000100:00100000:1.0:1767318777.565837:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1947 00000100:00100000:1.0:1767318777.565847:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985001856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318777.565995:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318777.566102:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985001856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 257us (823us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318777.566110:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1947 00010000:00010000:1.0:1767318777.691412:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318779.054813:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318779.100403:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318779.100407:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318779.100418:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985002240:0@lo:400:kworker.0 00000100:00100000:1.0:1767318779.100452:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318779.100488:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318779.100494:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985002496:0@lo:400:kworker.0 00000100:00100000:1.0:1767318779.100516:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318779.100542:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985002240 00000100:00100000:1.0:1767318779.100546:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985002240 00000100:00100000:1.0:1767318779.100577:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985002496 00000100:00100000:1.0:1767318779.100580:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985002496 00000100:00100000:1.0:1767318779.100599:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4224 00000100:00100000:1.0:1767318779.100606:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6add180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985002240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318779.100640:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6add180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985002240:12345-0@lo:400:kworker.0 Request processed in 35us (188us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318779.100646:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4224 00000100:00100000:1.0:1767318779.100659:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4225 00000100:00100000:1.0:1767318779.100665:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6add500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985002496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318779.100688:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6add500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985002496:12345-0@lo:400:kworker.0 Request processed in 24us (172us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318779.100694:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4225 00000100:00080000:1.0:1767318779.100716:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318779.100735:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985002496:0@lo:400:kworker.0 00000100:00100000:1.0:1767318779.100743:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985002240:0@lo:400:kworker.0 00000100:00100000:1.0:1767318779.101554:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985001984 00000100:00100000:1.0:1767318779.101559:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985001984 00000100:00100000:1.0:1767318779.101585:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985002368 00000100:00100000:1.0:1767318779.101588:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985002368 00000100:00100000:1.0:1767318779.101601:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985002624 00000100:00100000:1.0:1767318779.101603:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985002624 00000100:00100000:1.0:1767318779.101617:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4568 00000100:00100000:1.0:1767318779.101623:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985001984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318779.101641:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318779.101675:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985001984:12345-0@lo:38:kworker.0 Request processed in 51us (361us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318779.101681:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4568 00000100:00100000:1.0:1767318779.101693:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4569 00000100:00100000:1.0:1767318779.101697:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985002368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318779.101706:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318779.101727:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985002368:12345-0@lo:38:kworker.0 Request processed in 28us (307us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318779.101732:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4569 00000100:00100000:1.0:1767318779.101743:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4570 00000100:00100000:1.0:1767318779.101747:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985002624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318779.101755:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318779.101783:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985002624:12345-0@lo:38:kworker.0 Request processed in 35us (315us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318779.101788:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4570 00000100:00100000:3.0:1767318779.102216:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985002112 00000100:00100000:3.0:1767318779.102221:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985002112 00000100:00100000:3.0:1767318779.102254:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1632 00000100:00100000:3.0:1767318779.102260:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985002112:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318779.102271:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318779.102311:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985002112:12345-0@lo:38:kworker.0 Request processed in 50us (941us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318779.102317:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1632 00010000:00010000:1.0:1767318779.746807:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318782.683514:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1767318782.683537:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6addf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985003136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318782.683623:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318782.683697:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985003008:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318782.683719:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318782.683775:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985003136 00000100:00100000:1.0:1767318782.683783:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985003136 00000100:00100000:1.0:1767318782.683857:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1948 00000100:00100000:1.0:1767318782.683868:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985003136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318782.684077:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318782.684114:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985003136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 248us (490us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318782.684120:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1948 00000100:00100000:3.0:1767318782.684224:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6addf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985003136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318782.684237:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985003008:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318783.838135:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318784.169683:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318784.169728:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767318784.170496:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318784.220889:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985003520:0@lo:400:kworker.0 00000100:00100000:3.0:1767318784.220918:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318784.220940:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318784.220945:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985004160:0@lo:400:kworker.0 00000100:00100000:3.0:1767318784.220964:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318784.220990:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985003776 00000100:00100000:3.0:1767318784.220994:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985003776 00000100:00100000:3.0:1767318784.221020:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985003520 00000100:00100000:3.0:1767318784.221023:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985003520 00000100:00100000:3.0:1767318784.221045:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4227 00000100:00100000:3.0:1767318784.221052:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985003776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318784.221083:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985003776:12345-0@lo:400:kworker.0 Request processed in 31us (868us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318784.221089:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4227 00000100:00100000:3.0:1767318784.221102:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4228 00000100:00100000:3.0:1767318784.221108:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985003520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318784.221136:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985003520:12345-0@lo:400:kworker.0 Request processed in 29us (217us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318784.221142:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4228 00000100:00100000:3.0:1767318784.221161:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985004160 00000100:00100000:3.0:1767318784.221179:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985004160 00000100:00100000:3.0:1767318784.221201:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2806 00000100:00100000:3.0:1767318784.221206:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985004160:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318784.221376:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318784.221381:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318784.221382:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318784.221413:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985003264:0@lo:38:kworker.0 00000100:00100000:1.0:1767318784.221441:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318784.221467:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985003392:0@lo:38:kworker.0 00000100:00100000:1.0:1767318784.221492:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318784.221514:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985003648:0@lo:38:kworker.0 00000100:00100000:3.0:1767318784.225106:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985004160:12345-0@lo:400:kworker.0 Request processed in 27us (268us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318784.225116:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2806 00000100:00100000:3.0:1767318784.225155:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985003392 00000100:00100000:3.0:1767318784.225160:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985003392 00000100:00100000:3.0:1767318784.225208:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1633 00000100:00100000:3.0:1767318784.225213:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985003392:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318784.225223:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318784.225252:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985003392:12345-0@lo:38:kworker.0 Request processed in 38us (3759us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318784.225259:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1633 00000100:00100000:1.0:1767318784.225339:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318784.225389:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985003904:0@lo:38:kworker.0 00000100:00100000:1.0:1767318784.225420:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318784.225442:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985003648 00000100:00100000:1.0:1767318784.225446:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985003648 00000100:00100000:1.0:1767318784.225464:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985003904 00000100:00100000:1.0:1767318784.225466:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985003904 00000100:00100000:1.0:1767318784.225481:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4573 00000100:00100000:1.0:1767318784.225486:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985003648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318784.225496:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318784.225521:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985003648:12345-0@lo:38:kworker.0 Request processed in 34us (183us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318784.225526:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4573 00000100:00100000:1.0:1767318784.225537:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4574 00000100:00100000:1.0:1767318784.225540:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985003904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318784.225548:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318784.225568:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985003904:12345-0@lo:38:kworker.0 Request processed in 26us (148us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318784.225573:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4574 00000100:00080000:1.0:1767318784.225605:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318784.225610:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318784.225614:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985003904:0@lo:38:kworker.0 00000100:00080000:1.0:1767318784.225635:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318784.225639:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318784.225642:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985003264:0@lo:38:kworker.0 00000100:00080000:1.0:1767318784.225650:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318784.225653:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318784.225656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985003392:0@lo:38:kworker.0 00000100:00080000:1.0:1767318784.225663:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318784.225666:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318784.225669:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985003648:0@lo:38:kworker.0 00010000:00010000:1.0:1767318785.883401:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318787.806213:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985004416:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318787.806280:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318787.806347:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985004416 00000100:00100000:1.0:1767318787.806356:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985004416 00000100:00100000:1.0:1767318787.806416:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1950 00000100:00100000:1.0:1767318787.806426:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985004416:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318787.806452:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318787.806497:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985004416:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (218us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318787.806504:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1950 00000100:00100000:1.0:1767318787.806518:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985004288 00000100:00100000:1.0:1767318787.806521:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985004288 00000100:00100000:1.0:1767318787.806540:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1951 00000100:00100000:1.0:1767318787.806546:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985004288:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318787.806560:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318787.806589:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985004288:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (92us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318787.806595:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1951 00000400:00000100:3.0:1767318788.266457:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920524032 offset 0 length 520: 4 00000100:00100000:3.0:1767318788.266506:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318788.267518:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920524160 00000100:00100000:1.0:1767318788.267524:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920524160 00000100:00100000:1.0:1767318788.267562:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4575 00000100:00100000:1.0:1767318788.267567:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920524160:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318788.267588:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318788.267623:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920524160:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (279us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318788.267630:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4575 00000100:00100000:1.0:1767318788.315309:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318763/real 1767318763] req@ffff8ce1a6adf100 x1853163984997248/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318788 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318788.315478:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a6adf100 x1853163984997248/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318788 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318788.315504:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318788.315514:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318788.315518:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163984997248:0@lo:250:kworker.0 00000100:00100000:1.0:1767318788.315994:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985004928 00000100:00100000:1.0:1767318788.316002:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985004928 00000100:00100000:1.0:1767318788.316071:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4230 00000100:00100000:1.0:1767318788.316079:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985004928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318788.324677:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985004928:12345-0@lo:400:kworker.0 Request processed in 8595us (8745us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318788.324691:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4230 00000100:00100000:3.0:1767318788.327699:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318788.332677:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985005184 00000100:00100000:1.0:1767318788.332692:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985005184 00000100:00100000:1.0:1767318788.332798:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4231 00000100:00100000:1.0:1767318788.332810:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985005184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318788.332907:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985005184:12345-0@lo:400:kworker.0 Request processed in 100us (5212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318788.332915:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4231 00010000:00010000:1.0:1767318789.979414:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318792.027591:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318792.925693:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985005440 00000100:00100000:3.0:1767318792.925713:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985005440 00000100:00100000:3.0:1767318792.925782:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985005568 00000100:00100000:3.0:1767318792.925786:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985005568 00000100:00100000:3.0:1767318792.925816:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1952 00000100:00100000:3.0:1767318792.925832:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985005440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318792.925878:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318792.925947:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985005440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (1066us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318792.925956:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1952 00000100:00100000:3.0:1767318792.925983:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1953 00000100:00100000:3.0:1767318792.925993:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985005568:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318792.926017:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318792.926087:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985005568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (852us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318792.926093:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1953 00000100:00100000:3.0:1767318793.386508:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318793.386702:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920524416 00000100:00100000:1.0:1767318793.386767:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920524416 00000100:00100000:1.0:1767318793.386851:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4579 00000100:00100000:1.0:1767318793.386859:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920524416:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318793.386884:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318793.386954:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318793.387363:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920524416:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 135us (504us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318793.387371:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4579 00000100:00080000:1.0:1767318793.387399:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318793.435494:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318793.435504:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318793.435507:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318793.435511:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318793.435515:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318793.435523:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4036 00000100:00080000:3.0:1767318793.435527:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318793.435534:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318793.435581:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04e00 x1853163985005696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318793.435800:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318793.435804:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318793.435807:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318793.435811:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4036 00000100:00080000:3.0:1767318793.435813:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318793.435817:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318793.435833:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407800 x1853163985005824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318793.437107:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985005696 00000100:00100000:1.0:1767318793.437127:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985005696 00000100:00100000:1.0:1767318793.437396:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4580 00000100:00100000:1.0:1767318793.437408:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985005696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318793.437448:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318793.442565:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318793.442581:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318793.442584:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318793.442592:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4036 00000100:00080000:3.0:1767318793.442596:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318793.442602:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318793.442638:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407480 x1853163985006080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318793.442670:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318793.442673:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318793.442675:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318793.442679:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4036 00000100:00080000:3.0:1767318793.442681:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318793.442685:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318793.442707:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3480 x1853163985006336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318793.442741:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318793.442756:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985006208:0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.442814:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318793.442851:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318793.442860:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985005952:0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.442883:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318793.444654:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985005696:12345-0@lo:38:kworker.0 Request processed in 7237us (8919us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318793.444669:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4580 00000100:00080000:3.0:1767318793.444725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318793.444772:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985006208:0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.444800:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985005952:0@lo:400:kworker.0 00000100:00100000:1.0:1767318793.446813:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985006080 00000100:00100000:1.0:1767318793.446832:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985006080 00000100:00100000:1.0:1767318793.446927:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985006336 00000100:00100000:1.0:1767318793.446931:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985006336 00000100:00100000:1.0:1767318793.446976:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4581 00000100:00100000:1.0:1767318793.446987:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985006080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318793.447025:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318793.447815:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985006080:12345-0@lo:38:kworker.0 Request processed in 824us (1151us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318793.447832:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4581 00010000:00010000:1.0:1767318796.124470:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318798.171655:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318798.506423:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318798.506472:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318798.506605:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920524800 00000100:00100000:1.0:1767318798.506612:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920524800 00000100:00100000:1.0:1767318798.506673:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4583 00000100:00100000:1.0:1767318798.506683:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920524672:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318798.506716:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318798.506757:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920524672:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 74us (354us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767318798.506916:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318798.514152:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4583 00000100:00080000:3.0:1767318798.555285:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.555295:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318798.555299:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.555304:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.555308:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.555316:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.555321:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.555328:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318798.555368:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985006720/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318798.555398:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.555402:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.555404:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.555407:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.555410:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.555413:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318798.555426:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2680 x1853163985006848/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318798.555464:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.555467:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.555469:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.555472:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.555475:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.555478:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318798.555491:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3100 x1853163985007104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318798.555505:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.555508:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.555509:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.555512:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.555514:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.555517:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318798.555525:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3800 x1853163985007360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318798.555548:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985007232:0@lo:400:kworker.0 00000100:00100000:1.0:1767318798.561723:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985006976 00000100:00100000:1.0:1767318798.561736:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985006976 00000100:00100000:1.0:1767318798.561824:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4236 00000100:00100000:1.0:1767318798.561833:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985006976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318798.561929:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985006976:12345-0@lo:400:kworker.0 Request processed in 97us (1801us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318798.561936:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4236 00000100:00100000:1.0:1767318798.561987:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985006976:0@lo:400:kworker.0 00000100:00100000:3.0:1767318798.562570:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318798.563302:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985007232 00000100:00100000:1.0:1767318798.563309:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985007232 00000100:00100000:1.0:1767318798.563649:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4237 00000100:00100000:1.0:1767318798.563658:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985007232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318798.563734:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985007232:12345-0@lo:400:kworker.0 Request processed in 77us (1164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318798.563741:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4237 00000100:00100000:3.0:1767318798.564478:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985007232:0@lo:400:kworker.0 00000100:00100000:3.0:1767318798.578267:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985006720 00000100:00100000:3.0:1767318798.578276:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985006720 00000100:00100000:3.0:1767318798.578308:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985007360 00000100:00100000:3.0:1767318798.578313:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985007360 00000100:00100000:3.0:1767318798.584532:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4585 00000100:00100000:3.0:1767318798.584545:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985007104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318798.584570:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318798.584674:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985007104:12345-0@lo:38:kworker.0 Request processed in 129us (6521us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318798.584682:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4585 00000100:00100000:3.0:1767318798.584702:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4586 00000100:00100000:3.0:1767318798.584706:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985007360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318798.584716:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318798.584740:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985007360:12345-0@lo:38:kworker.0 Request processed in 32us (6503us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318798.584745:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4586 00000100:00100000:3.0:1767318798.584790:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985006848 00000100:00100000:3.0:1767318798.584798:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985006848 00000100:00100000:3.0:1767318798.584826:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1636 00000100:00100000:3.0:1767318798.584830:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985006848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318798.584839:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318798.584864:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985006848:12345-0@lo:38:kworker.0 Request processed in 32us (6760us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318798.584869:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1636 00000100:00080000:3.0:1767318798.584922:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318798.584928:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318798.584931:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985006720:0@lo:38:kworker.0 00000100:00080000:3.0:1767318798.584942:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318798.584946:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318798.584949:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985006848:0@lo:38:kworker.0 00000100:00080000:3.0:1767318798.584957:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318798.584960:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318798.584963:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985007104:0@lo:38:kworker.0 00000100:00080000:3.0:1767318798.584971:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318798.584974:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318798.584977:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985007360:0@lo:38:kworker.0 00010000:00010000:1.0:1767318802.267368:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318803.164309:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318803.164320:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985007616:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318803.164370:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318803.165227:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1957 00000100:00100000:1.0:1767318803.165236:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985007616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318803.165253:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318803.165300:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985007616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (929us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318803.165309:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1957 00000100:00100000:1.0:1767318803.165387:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985007616:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318803.166033:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985007488:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318803.291449:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318803.676404:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318803.676415:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318803.676418:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318803.676422:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318803.676426:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318803.676434:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4046 00000100:00080000:1.0:1767318803.676439:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318803.676446:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318803.676484:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6a00 x1853163985007744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318803.676521:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318803.676524:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318803.676526:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318803.676529:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4046 00000100:00080000:1.0:1767318803.676532:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318803.676535:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318803.676551:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6680 x1853163985007872/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318803.676588:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318803.676592:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318803.676593:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318803.676597:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4046 00000100:00080000:1.0:1767318803.676599:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318803.676601:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318803.676612:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5180 x1853163985008128/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318803.676626:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318803.676628:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318803.676630:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318803.676633:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4046 00000100:00080000:1.0:1767318803.676635:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318803.676638:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318803.676650:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adf100 x1853163985008384/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318803.678461:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985007744:0@lo:38:kworker.0 00000100:00100000:3.0:1767318803.678519:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318803.678594:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985007872:0@lo:38:kworker.0 00000100:00100000:3.0:1767318803.678624:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318803.678657:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985008128:0@lo:38:kworker.0 00000100:00100000:3.0:1767318803.678688:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318803.678704:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985008384:0@lo:38:kworker.0 00000100:00100000:3.0:1767318803.678723:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318803.678756:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985007744 00000100:00100000:3.0:1767318803.678762:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985007744 00000100:00100000:3.0:1767318803.678786:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985008128 00000100:00100000:3.0:1767318803.678790:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985008128 00000100:00100000:3.0:1767318803.678801:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985008384 00000100:00100000:3.0:1767318803.678804:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985008384 00000100:00100000:3.0:1767318803.678819:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4588 00000100:00100000:3.0:1767318803.678825:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985007744:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318803.678844:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318803.678879:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985007744:12345-0@lo:38:kworker.0 Request processed in 53us (360us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318803.678885:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4588 00000100:00100000:3.0:1767318803.678899:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4589 00000100:00100000:3.0:1767318803.678903:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985008128:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318803.678911:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318803.678932:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985008128:12345-0@lo:38:kworker.0 Request processed in 28us (244us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318803.678937:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4589 00000100:00100000:3.0:1767318803.678948:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4590 00000100:00100000:3.0:1767318803.678951:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985008384:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318803.678959:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318803.678979:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985008384:12345-0@lo:38:kworker.0 Request processed in 26us (255us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318803.678986:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4590 00000100:00080000:3.0:1767318803.679022:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318803.679027:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318803.679031:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985007744:0@lo:38:kworker.0 00000100:00080000:3.0:1767318803.679040:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318803.679044:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318803.679046:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985008128:0@lo:38:kworker.0 00000100:00080000:3.0:1767318803.679053:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318803.679055:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318803.679058:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985008384:0@lo:38:kworker.0 00000100:00080000:1.0:1767318803.679824:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318803.679829:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318803.679832:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985007872:0@lo:38:kworker.0 00010000:00010000:1.0:1767318804.315475:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318808.283703:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318808.283711:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985008640:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318808.283748:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318808.284003:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985008640 00000100:00100000:3.0:1767318808.284011:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985008640 00000100:00100000:3.0:1767318808.284086:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985008512 00000100:00100000:3.0:1767318808.284090:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985008512 00000100:00100000:3.0:1767318808.284112:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1958 00000100:00100000:3.0:1767318808.284121:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985008640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318808.284149:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318808.284220:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985008640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (471us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318808.284228:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1958 00000100:00100000:3.0:1767318808.284246:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1959 00000100:00100000:3.0:1767318808.284252:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985008512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318808.284267:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318808.284303:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985008512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (398us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318808.284309:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1959 00010000:00010000:1.0:1767318808.413325:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318808.798915:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985008768:0@lo:38:kworker.0 00000100:00100000:3.0:1767318808.798981:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318808.799899:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985008768 00000100:00100000:1.0:1767318808.799909:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985008768 00000100:00100000:1.0:1767318808.799960:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4592 00000100:00100000:1.0:1767318808.799967:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985008768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318808.799990:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318808.802571:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985008896:0@lo:38:kworker.0 00000100:00100000:3.0:1767318808.802643:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318808.810408:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985008768:12345-0@lo:38:kworker.0 Request processed in 10418us (11404us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318808.810417:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4592 00000100:00080000:3.0:1767318808.816806:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318808.816820:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318808.816825:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985008768:0@lo:38:kworker.0 00000100:00100000:3.0:1767318808.816866:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985009152:0@lo:38:kworker.0 00000100:00100000:3.0:1767318808.816927:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318808.816971:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985009408:0@lo:38:kworker.0 00000100:00100000:3.0:1767318808.816992:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318808.817025:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318808.817029:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318808.817032:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985008896:0@lo:38:kworker.0 00000100:00080000:3.0:1767318808.818362:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318808.818368:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318808.818372:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985009152:0@lo:38:kworker.0 00000100:00080000:3.0:1767318808.818382:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318808.818385:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318808.818388:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985009408:0@lo:38:kworker.0 00010000:00010000:1.0:1767318809.437572:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318810.460330:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318813.411915:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985009792:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318813.411959:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318813.411973:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985009920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318813.412453:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318813.412620:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985009920 00000100:00100000:1.0:1767318813.412628:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985009920 00000100:00100000:1.0:1767318813.412785:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1961 00000100:00100000:1.0:1767318813.412795:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985009920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318813.412826:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318813.412874:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985009920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (422us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318813.412881:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1961 00000100:00100000:1.0:1767318813.412915:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985009920:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318813.867400:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318813.867445:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767318813.873214:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920525696 offset 0 length 520: 4 00000100:00080000:3.0:1767318813.875948:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318813.917804:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318788/real 1767318788] req@ffff8ce1b351b800 x1853163985004544/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318813 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318813.917842:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b351b800 x1853163985004544/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318813 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318813.917884:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.917907:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318813.917912:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985004544:0@lo:250:kworker.0 00000100:00100000:3.0:1767318813.918863:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985010048:0@lo:250:kworker.0 00000400:00000100:3.0:1767318813.918918:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985010048 offset 0 length 520: 4 00000100:00100000:3.0:1767318813.919710:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985010176:0@lo:38:kworker.0 00000100:00100000:1.0:1767318813.919965:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985010944 00000100:00100000:1.0:1767318813.919972:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985010944 00000100:00100000:1.0:1767318813.920019:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985011072 00000100:00100000:1.0:1767318813.920023:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985011072 00000100:00100000:1.0:1767318813.920047:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2809 00000100:00100000:1.0:1767318813.920060:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e376d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985010944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318813.920075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318813.920134:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985010304:0@lo:38:kworker.0 00000100:00100000:1.0:1767318813.920181:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e376d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985010944:12345-0@lo:400:kworker.0 Request processed in 105us (871us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318813.920192:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2809 00000100:00100000:1.0:1767318813.920208:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2810 00000100:00100000:1.0:1767318813.920214:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e374700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985011072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318813.925120:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e374700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985011072:12345-0@lo:400:kworker.0 Request processed in 4902us (5774us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318813.925136:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2810 00000100:00100000:1.0:1767318813.925215:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4596 00000100:00100000:1.0:1767318813.925221:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985010176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318813.925248:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318813.925284:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985010176:12345-0@lo:38:kworker.0 Request processed in 61us (5223us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318813.925290:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4596 00000100:00080000:1.0:1767318813.925403:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318813.925467:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985011072:0@lo:400:kworker.0 00000100:00100000:1.0:1767318813.925483:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985010944:0@lo:400:kworker.0 00000100:00100000:3.0:1767318813.927506:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318813.927580:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985010560:0@lo:38:kworker.0 00000100:00100000:1.0:1767318813.928241:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985010688:0@lo:400:kworker.0 00000100:00100000:1.0:1767318813.928272:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985010432:0@lo:400:kworker.0 00000100:00100000:3.0:1767318813.937716:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318813.938980:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985010816:0@lo:38:kworker.0 00000100:00100000:3.0:1767318813.939051:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318813.947147:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.947183:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318813.947189:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985010176:0@lo:38:kworker.0 00000100:00080000:3.0:1767318813.947208:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.947212:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318813.947215:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985010304:0@lo:38:kworker.0 00000100:00100000:1.0:1767318813.948510:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985010560 00000100:00100000:1.0:1767318813.948528:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985010560 00000100:00100000:1.0:1767318813.948593:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985010816 00000100:00100000:1.0:1767318813.948598:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985010816 00000100:00100000:1.0:1767318813.948641:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4597 00000100:00100000:1.0:1767318813.948651:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985010560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318813.948676:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318813.948764:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985010560:12345-0@lo:38:kworker.0 Request processed in 114us (11059us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318813.948771:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4597 00000100:00100000:1.0:1767318813.948790:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4598 00000100:00100000:1.0:1767318813.948794:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985010816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318813.948814:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318813.948843:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985010816:12345-0@lo:38:kworker.0 Request processed in 47us (9792us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318813.948850:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4598 00000100:00080000:3.0:1767318813.951019:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.951036:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318813.951042:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985010560:0@lo:38:kworker.0 00000100:00080000:3.0:1767318813.951057:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.951061:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318813.951064:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985010816:0@lo:38:kworker.0 00010000:00010000:1.0:1767318814.556844:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318815.580009:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318816.603887:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318818.523627:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318818.523646:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985011200:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318818.523770:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318818.524336:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318818.527887:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318818.527910:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985011328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318818.527972:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318818.528094:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985011328 00000100:00100000:1.0:1767318818.528104:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985011328 00000100:00080000:3.0:1767318818.528191:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318818.528209:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1963 00000100:00100000:1.0:1767318818.528229:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985011328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318818.528261:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318818.528325:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985011328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (351us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318818.528334:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1963 00000100:00100000:1.0:1767318818.528396:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985011328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318818.998416:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318818.998518:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:3.0:1767318818.999278:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318819.036088:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318819.036099:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318819.036103:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318819.036107:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318819.036111:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318819.036119:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4062 00000100:00080000:3.0:1767318819.036124:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318819.036130:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318819.036189:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594e00 x1853163985011456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318819.036227:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318819.036231:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318819.036233:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318819.036236:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4062 00000100:00080000:3.0:1767318819.036239:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318819.036243:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318819.036256:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594a80 x1853163985011584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318819.036310:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318819.036314:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318819.036315:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318819.036356:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4062 00000100:00080000:3.0:1767318819.036359:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318819.036362:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318819.036375:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595f80 x1853163985011840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318819.036390:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318819.036393:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318819.036394:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318819.036398:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4062 00000100:00080000:3.0:1767318819.036400:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318819.036402:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318819.036411:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596680 x1853163985012096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318819.037324:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985012224:0@lo:400:kworker.0 00000100:00100000:1.0:1767318819.037361:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318819.037395:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318819.037400:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985012352:0@lo:400:kworker.0 00000100:00100000:1.0:1767318819.037420:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318819.037444:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985012224 00000100:00100000:1.0:1767318819.037447:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985012224 00000100:00100000:1.0:1767318819.037472:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985012352 00000100:00100000:1.0:1767318819.037475:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985012352 00000100:00100000:1.0:1767318819.037491:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2811 00000100:00100000:1.0:1767318819.037512:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985012224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318819.037573:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985012224:12345-0@lo:400:kworker.0 Request processed in 62us (210us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318819.037579:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2811 00000100:00100000:1.0:1767318819.037591:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2812 00000100:00100000:1.0:1767318819.037597:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a07050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985012352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318819.037622:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a07050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985012352:12345-0@lo:400:kworker.0 Request processed in 26us (201us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318819.037628:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2812 00000100:00100000:1.0:1767318819.037842:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985012352:0@lo:400:kworker.0 00000100:00100000:1.0:1767318819.037852:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985012224:0@lo:400:kworker.0 00010000:00010000:1.0:1767318820.700623:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318821.727015:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318822.747630:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318824.109304:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318824.111584:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920526080 00000100:00100000:1.0:1767318824.111599:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920526080 00000100:00100000:1.0:1767318824.111676:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4603 00000100:00100000:1.0:1767318824.111684:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920526080:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318824.111708:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318824.112324:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920526080:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 636us (3029us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318824.112333:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4603 00000100:00100000:3.0:1767318824.113492:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318824.114875:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920526208 00000100:00100000:1.0:1767318824.114889:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920526208 00000100:00100000:1.0:1767318824.114975:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4250 00000100:00100000:1.0:1767318824.114985:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920526208:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318824.115074:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920526208:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (1586us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318824.115081:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4250 00000100:00080000:1.0:1767318824.156341:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318824.156352:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318824.156355:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318824.156358:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318824.156363:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318824.156372:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318824.156376:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318824.156382:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318824.156421:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb800 x1853163985012736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318824.156453:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318824.156457:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318824.156459:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318824.156463:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318824.156466:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318824.156469:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318824.156482:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8000 x1853163985012864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318824.156513:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318824.156517:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318824.156519:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318824.156522:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318824.156525:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318824.156528:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318824.156539:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdaa00 x1853163985013120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318824.156554:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318824.156557:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318824.156559:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318824.156562:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318824.156565:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318824.156568:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318824.156578:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb480 x1853163985013376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318824.156631:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985013504:0@lo:400:kworker.0 00000100:00100000:3.0:1767318824.156761:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985012736:0@lo:38:kworker.0 00000100:00100000:1.0:1767318824.157800:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318824.157863:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318824.157872:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985013632:0@lo:400:kworker.0 00000100:00100000:1.0:1767318824.157901:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318824.157935:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985013504 00000100:00100000:1.0:1767318824.157943:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985013504 00000100:00100000:1.0:1767318824.157983:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985013632 00000100:00100000:1.0:1767318824.157987:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985013632 00000100:00100000:1.0:1767318824.158006:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2813 00000100:00100000:1.0:1767318824.158014:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985013504:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318824.158062:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985013504:12345-0@lo:400:kworker.0 Request processed in 48us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318824.158070:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2813 00000100:00100000:1.0:1767318824.158084:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2814 00000100:00100000:1.0:1767318824.158090:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985013632:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318824.158128:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985013632:12345-0@lo:400:kworker.0 Request processed in 38us (225us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318824.158144:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2814 00000100:00100000:1.0:1767318824.158371:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985013632:0@lo:400:kworker.0 00000100:00100000:1.0:1767318824.158388:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985013504:0@lo:400:kworker.0 00000100:00100000:1.0:1767318824.160456:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985012992 00000100:00100000:1.0:1767318824.160465:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985012992 00000100:00100000:1.0:1767318824.160528:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4251 00000100:00100000:1.0:1767318824.160538:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985012992:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318824.160595:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985012992:12345-0@lo:400:kworker.0 Request processed in 59us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318824.160602:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4251 00000100:00100000:3.0:1767318824.167602:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318824.167643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985012864:0@lo:38:kworker.0 00000100:00100000:3.0:1767318824.167681:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318824.167709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985013120:0@lo:38:kworker.0 00000100:00100000:3.0:1767318824.167732:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318824.167748:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985013376:0@lo:38:kworker.0 00000100:00100000:3.0:1767318824.167767:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318824.167850:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985012864 00000100:00100000:3.0:1767318824.167858:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985012864 00000100:00100000:3.0:1767318824.167895:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1641 00000100:00100000:3.0:1767318824.167901:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985012864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318824.167923:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318824.167964:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985012864:12345-0@lo:38:kworker.0 Request processed in 61us (281us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318824.167972:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1641 00010000:00010000:1.0:1767318826.845477:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318827.869052:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318828.763894:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318828.763919:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985013760:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318828.763988:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:0.0:1767318828.764762:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318828.764882:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985013760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 155us (893us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318828.764889:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1966 00000100:00100000:3.0:1767318828.765740:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318828.765751:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce090003100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985013888:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318828.765803:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318828.775023:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985013888 00000100:00100000:1.0:1767318828.775054:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985013888 00000100:00100000:1.0:1767318828.775292:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1967 00000100:00100000:1.0:1767318828.775305:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090000e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985013888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318828.775474:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318828.775655:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090000e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985013888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 353us (9851us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318828.775662:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1967 00000100:00100000:1.0:1767318828.775736:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090003100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985013888:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767318828.776005:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318828.891651:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318829.228093:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318829.228142:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318829.228356:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920526336 00000100:00100000:1.0:1767318829.228367:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920526336 00000100:00100000:1.0:1767318829.228411:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920526464 00000100:00100000:1.0:1767318829.228415:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920526464 00000100:00100000:1.0:1767318829.228470:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4253 00000100:00100000:1.0:1767318829.228482:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090001880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920526464:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318829.228562:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090001880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920526464:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (419us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318829.228570:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4253 00000100:00100000:1.0:1767318829.228590:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4607 00000100:00100000:1.0:1767318829.228594:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090003b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920526336:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318829.228614:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318829.228640:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090003b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920526336:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (558us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318829.228646:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4607 00000100:00080000:1.0:1767318829.228661:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318829.277886:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985014272:0@lo:400:kworker.0 00000100:00100000:3.0:1767318829.277897:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985014528:0@lo:400:kworker.0 00000100:00100000:0.0:1767318829.277974:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985014016:0@lo:38:kworker.0 00000100:00100000:1.0:1767318829.278035:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318829.278059:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985014272 00000100:00100000:3.0:1767318829.278067:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985014272 00000100:00100000:3.0:1767318829.278121:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4254 00000100:00100000:3.0:1767318829.278130:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985014272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318829.278207:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985014272:12345-0@lo:400:kworker.0 Request processed in 77us (171us total) trans 0 rc 0/0 00000100:00080000:1.0:1767318829.288193:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318829.288515:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4254 00000100:00100000:0.0:1767318829.291011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318829.291195:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985014016 00000100:00100000:3.0:1767318829.291208:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985014016 00000100:00100000:3.0:1767318829.291250:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4255 00000100:00100000:3.0:1767318829.291266:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985014528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318829.292098:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4608 00000100:00100000:1.0:1767318829.292108:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985014016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318829.292133:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318829.292210:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985014016:12345-0@lo:38:kworker.0 Request processed in 102us (1206us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318829.292217:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4608 00000100:00100000:3.0:1767318829.292467:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985014528:12345-0@lo:400:kworker.0 Request processed in 1207us (14308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318829.292474:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4255 00000100:00100000:1.0:1767318829.292600:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985014528:0@lo:400:kworker.0 00000100:00080000:0.0:1767318829.295557:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318829.295569:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318829.295575:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985014016:0@lo:38:kworker.0 00000100:00100000:0.0:1767318829.295587:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985014144:0@lo:38:kworker.0 00000100:00100000:0.0:1767318829.295647:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318829.295681:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985014400:0@lo:38:kworker.0 00000100:00100000:0.0:1767318829.295706:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318829.295728:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985014656:0@lo:38:kworker.0 00000100:00100000:0.0:1767318829.295754:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318829.295825:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985014400 00000100:00100000:0.0:1767318829.295832:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985014400 00000100:00100000:0.0:1767318829.295863:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985014656 00000100:00100000:0.0:1767318829.295866:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985014656 00000100:00100000:0.0:1767318829.295886:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4609 00000100:00100000:0.0:1767318829.295893:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985014400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318829.295914:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318829.295954:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985014400:12345-0@lo:38:kworker.0 Request processed in 60us (246us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318829.295962:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4609 00000100:00100000:0.0:1767318829.295979:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4610 00000100:00100000:0.0:1767318829.295983:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ada300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985014656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318829.295993:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318829.296026:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ada300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985014656:12345-0@lo:38:kworker.0 Request processed in 30us (261us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318829.296032:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4610 00000100:00080000:0.0:1767318829.296063:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318829.296068:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318829.296071:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985014400:0@lo:38:kworker.0 00000100:00080000:0.0:1767318829.296079:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318829.296082:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318829.296085:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985014656:0@lo:38:kworker.0 00000100:00100000:1.0:1767318829.297453:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985014144 00000100:00100000:1.0:1767318829.297462:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985014144 00000100:00100000:1.0:1767318829.297501:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1642 00000100:00100000:1.0:1767318829.297507:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ada680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985014144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318829.297535:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318829.297591:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ada680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985014144:12345-0@lo:38:kworker.0 Request processed in 83us (1944us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318829.297598:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1642 00000100:00080000:0.0:1767318829.300140:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318829.300153:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318829.300157:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985014144:0@lo:38:kworker.0 00010000:00010000:0.0:1767318829.917570:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318832.988206:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318833.883598:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985014912:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318833.883699:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318833.883825:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985014912 00000100:00100000:0.0:1767318833.883837:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985014912 00000100:00100000:0.0:1767318833.883939:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1968 00000100:00100000:0.0:1767318833.883951:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985014912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318833.883991:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767318833.884076:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318833.886418:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985014784 00000100:00100000:3.0:1767318833.886429:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985014784 00000100:00100000:3.0:1767318833.886521:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1969 00000100:00100000:3.0:1767318833.886532:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985014784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318833.886572:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318833.886639:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985014784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (413us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318833.886647:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1969 00000100:00080000:1.0:1767318833.886899:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318833.886990:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090001f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985014784:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318833.893712:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985014912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 9749us (9997us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318833.893732:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1968 00000100:00100000:1.0:1767318833.893910:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985014912:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318834.014563:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318834.363001:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920526720 00000100:00100000:3.0:1767318834.363016:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920526720 00000100:00100000:3.0:1767318834.363092:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920526592 00000100:00100000:3.0:1767318834.363101:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920526592 00000100:00100000:3.0:1767318834.363119:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4256 00000100:00100000:3.0:1767318834.363128:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920526720:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318834.363245:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920526720:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 119us (545us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318834.363251:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4256 00000100:00100000:3.0:1767318834.363268:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4611 00000100:00100000:3.0:1767318834.363271:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920526592:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318834.363293:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318834.363312:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920526592:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (540us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318834.363316:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4611 00000100:00080000:1.0:1767318834.363706:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318834.396305:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985015040:0@lo:38:kworker.0 00000100:00100000:0.0:1767318834.396398:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318834.396477:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985015168:0@lo:38:kworker.0 00000100:00100000:0.0:1767318834.396504:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318834.396530:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985015424:0@lo:38:kworker.0 00000100:00100000:0.0:1767318834.396553:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318834.396572:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985015680:0@lo:38:kworker.0 00000100:00100000:0.0:1767318834.396594:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318834.396635:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985015168 00000100:00100000:0.0:1767318834.396644:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985015168 00000100:00100000:0.0:1767318834.396684:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1643 00000100:00100000:0.0:1767318834.396692:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985015168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318834.396717:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318834.396765:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985015168:12345-0@lo:38:kworker.0 Request processed in 72us (258us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318834.396773:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1643 00000100:00100000:0.0:1767318834.396799:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985015040 00000100:00100000:0.0:1767318834.396803:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985015040 00000100:00100000:0.0:1767318834.396825:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985015424 00000100:00100000:0.0:1767318834.396828:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985015424 00000100:00100000:0.0:1767318834.396840:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985015680 00000100:00100000:0.0:1767318834.396842:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985015680 00000100:00100000:0.0:1767318834.396856:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4612 00000100:00100000:0.0:1767318834.396875:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985015040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318834.396884:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318834.396911:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985015040:12345-0@lo:38:kworker.0 Request processed in 50us (513us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318834.396917:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4612 00000100:00100000:0.0:1767318834.396939:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4613 00000100:00100000:0.0:1767318834.396944:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985015424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318834.396952:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318834.396976:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985015424:12345-0@lo:38:kworker.0 Request processed in 29us (421us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318834.396981:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4613 00000100:00100000:0.0:1767318834.396991:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4614 00000100:00100000:0.0:1767318834.396995:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985015680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318834.397002:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318834.397024:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985015680:12345-0@lo:38:kworker.0 Request processed in 27us (429us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318834.397029:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4614 00000100:00080000:0.0:1767318834.397089:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318834.397095:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318834.397098:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985015040:0@lo:38:kworker.0 00000100:00080000:0.0:1767318834.397107:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318834.397110:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318834.397113:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985015168:0@lo:38:kworker.0 00000100:00080000:0.0:1767318834.397119:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318834.397122:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318834.397125:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985015424:0@lo:38:kworker.0 00000100:00080000:0.0:1767318834.397131:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318834.397134:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318834.397137:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985015680:0@lo:38:kworker.0 00000100:00100000:3.0:1767318834.397595:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985015296:0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.397642:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318834.397676:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318834.397684:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985015552:0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.397707:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318834.398123:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318834.398153:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985015552:0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.398184:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985015296:0@lo:400:kworker.0 00010000:00010000:1.0:1767318835.041684:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318836.060484:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767318838.445110:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920526848 offset 0 length 520: 4 00000100:00100000:3.0:1767318838.445159:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318838.445243:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318838.447057:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920526976 00000100:00100000:1.0:1767318838.447069:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920526976 00000100:00100000:1.0:1767318838.447124:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920527104 00000100:00100000:1.0:1767318838.447128:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920527104 00000100:00100000:1.0:1767318838.447290:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4259 00000100:00100000:1.0:1767318838.447303:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090002300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920527104:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318838.447382:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090002300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920527104:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (2138us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318838.447389:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4259 00000100:00100000:1.0:1767318838.447406:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4615 00000100:00100000:1.0:1767318838.447410:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090001500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920526976:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318838.447429:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318838.447465:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090001500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920526976:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (2294us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318838.447472:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4615 00000100:00080000:3.0:1767318838.447520:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318838.491487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318838.491493:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767318838.491496:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318838.491501:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4062 00000100:00080000:0.0:1767318838.491505:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318838.491513:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318838.491555:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985015808/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318838.491577:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318838.491580:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318838.491582:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318838.491585:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4082 00000100:00080000:0.0:1767318838.491587:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318838.491591:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318838.491612:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b100 x1853163985015936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318838.491620:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318838.491623:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318838.491625:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318838.491628:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4082 00000100:00080000:0.0:1767318838.491630:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318838.491632:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318838.491642:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519f80 x1853163985016064/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318838.491683:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318838.491687:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318838.491688:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318838.491692:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4082 00000100:00080000:0.0:1767318838.491694:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318838.491697:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318838.491707:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519880 x1853163985016320/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318838.491722:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318838.491725:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318838.491726:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318838.491729:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4082 00000100:00080000:0.0:1767318838.491731:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318838.491734:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318838.491742:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351ad80 x1853163985016576/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318838.491799:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985016704:0@lo:400:kworker.0 00000100:00100000:1.0:1767318838.491887:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985016448:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.491891:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318838.491919:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318838.491961:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318838.491967:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985016832:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.491988:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318838.492017:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985016704 00000100:00100000:0.0:1767318838.492024:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985016704 00000100:00100000:0.0:1767318838.492066:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985016832 00000100:00100000:0.0:1767318838.492069:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985016832 00000100:00100000:0.0:1767318838.492086:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2815 00000100:00100000:0.0:1767318838.492093:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985016704:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.492136:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985016704:12345-0@lo:400:kworker.0 Request processed in 43us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318838.492142:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2815 00000100:00100000:0.0:1767318838.492156:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2816 00000100:00100000:0.0:1767318838.492182:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985016832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.492209:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985016832:12345-0@lo:400:kworker.0 Request processed in 47us (220us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318838.492214:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2816 00000100:00080000:0.0:1767318838.492239:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318838.492242:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318838.492266:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985016832:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.492502:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985016704:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.492539:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985016448 00000100:00100000:0.0:1767318838.492543:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985016448 00000100:00100000:0.0:1767318838.492569:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4260 00000100:00100000:0.0:1767318838.492575:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985016448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318838.492706:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985016192:0@lo:400:kworker.0 00000100:00100000:3.0:1767318838.497336:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318838.497453:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985016192 00000100:00100000:3.0:1767318838.497463:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985016192 00000100:00100000:3.0:1767318838.497513:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4261 00000100:00100000:3.0:1767318838.497521:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090003800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985016192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318838.497576:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090003800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985016192:12345-0@lo:400:kworker.0 Request processed in 56us (246us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318838.497585:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4261 00000100:00100000:3.0:1767318838.497637:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985016192:0@lo:400:kworker.0 00000100:00080000:3.0:1767318838.497679:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318838.503369:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985016448:12345-0@lo:400:kworker.0 Request processed in 10783us (11438us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318838.503384:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4260 00000100:00100000:1.0:1767318838.508152:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985016448:0@lo:400:kworker.0 00000100:00100000:1.0:1767318839.003578:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985017088:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318839.003649:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318839.003706:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318839.003714:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce090000000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985016960:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318839.003738:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318839.003869:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985017088 00000100:00100000:3.0:1767318839.003878:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985017088 00000100:00100000:3.0:1767318839.003944:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985016960 00000100:00100000:3.0:1767318839.003947:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985016960 00000100:00100000:3.0:1767318839.003967:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1970 00000100:00100000:3.0:1767318839.003975:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985017088:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318839.004009:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318839.004057:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985017088:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (407us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318839.004064:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1970 00000100:00100000:3.0:1767318839.004082:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1971 00000100:00100000:3.0:1767318839.004089:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985016960:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318839.004105:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318839.004131:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985016960:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (393us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318839.004137:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1971 00000100:00100000:1.0:1767318839.004147:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985017088:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318839.004193:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090000000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985016960:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318839.131937:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318840.155640:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318841.180045:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318842.206131:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318843.585769:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318843.586721:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318843.603022:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920527360 00000100:00100000:1.0:1767318843.606274:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920527360 00000100:00080000:0.0:1767318843.606507:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318843.607248:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920527232 00000100:00100000:1.0:1767318843.607261:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920527232 00000100:00100000:3.0:1767318843.607366:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4262 00000100:00100000:3.0:1767318843.607389:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920527360:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318843.607523:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920527360:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 142us (21791us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318843.607532:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4262 00000100:00100000:3.0:1767318843.607557:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4619 00000100:00100000:3.0:1767318843.607562:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920527232:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318843.607588:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318843.607629:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920527232:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 65us (20912us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318843.607644:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4619 00000100:00080000:0.0:1767318843.611446:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318843.611461:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318843.611467:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318843.611472:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318843.611477:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318843.611490:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4086 00000100:00080000:0.0:1767318843.611496:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318843.611507:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318843.611570:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7b80 x1853163985017216/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318843.615611:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318843.615628:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318843.615635:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318843.615647:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4086 00000100:00080000:0.0:1767318843.615654:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318843.615666:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318843.615745:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351bb80 x1853163985017344/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318843.615814:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318843.615819:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318843.615821:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318843.615825:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4086 00000100:00080000:0.0:1767318843.615828:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318843.615832:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318843.615857:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519880 x1853163985017600/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318843.615882:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318843.615886:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318843.615888:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318843.615892:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4086 00000100:00080000:0.0:1767318843.615894:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318843.615897:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318843.615911:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b100 x1853163985017856/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318843.616005:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985017472:0@lo:400:kworker.0 00000100:00100000:0.0:1767318843.616117:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318843.641827:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.642315:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985017472 00000100:00100000:1.0:1767318843.642330:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985017472 00000100:00100000:1.0:1767318843.642428:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4263 00000100:00100000:1.0:1767318843.642440:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985017472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318843.642559:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985017472:12345-0@lo:400:kworker.0 Request processed in 122us (26442us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318843.642576:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4263 00000100:00100000:1.0:1767318843.642647:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985017472:0@lo:400:kworker.0 00000100:00080000:1.0:1767318843.642733:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318843.653681:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.653777:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985017344:0@lo:38:kworker.0 00000100:00100000:1.0:1767318843.654026:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.654268:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985017600:0@lo:38:kworker.0 00000100:00100000:1.0:1767318843.654302:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.654325:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985017856:0@lo:38:kworker.0 00000100:00100000:1.0:1767318843.654350:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.654403:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985017728 00000100:00100000:1.0:1767318843.654414:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985017728 00000100:00100000:1.0:1767318843.654490:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985017216 00000100:00100000:1.0:1767318843.654494:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985017216 00000100:00100000:1.0:1767318843.654511:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985017600 00000100:00100000:1.0:1767318843.654514:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985017600 00000100:00100000:1.0:1767318843.654522:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985017856 00000100:00100000:1.0:1767318843.654524:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985017856 00000100:00100000:1.0:1767318843.654538:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4264 00000100:00100000:1.0:1767318843.654549:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985017728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318843.654619:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985017728:12345-0@lo:400:kworker.0 Request processed in 71us (12815us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318843.654628:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4264 00000100:00100000:1.0:1767318843.654648:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4620 00000100:00100000:1.0:1767318843.654653:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985017216:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318843.654678:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318843.654717:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985017216:12345-0@lo:38:kworker.0 Request processed in 62us (1050us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318843.654724:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4620 00000100:00100000:1.0:1767318843.654737:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4621 00000100:00100000:1.0:1767318843.654741:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985017600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318843.654766:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318843.654791:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985017600:12345-0@lo:38:kworker.0 Request processed in 48us (488us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318843.654796:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4621 00000100:00100000:3.0:1767318843.655436:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4622 00000100:00100000:3.0:1767318843.655444:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985017856:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318843.655456:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318843.655493:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985017856:12345-0@lo:38:kworker.0 Request processed in 49us (1142us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318843.655500:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4622 00000100:00100000:3.0:1767318843.655574:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985017728:0@lo:400:kworker.0 00000100:00080000:3.0:1767318843.655633:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318843.661860:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.661882:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.661889:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985017216:0@lo:38:kworker.0 00000100:00080000:0.0:1767318843.661910:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.661914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.661918:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985017344:0@lo:38:kworker.0 00000100:00080000:0.0:1767318843.661930:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.661933:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.661936:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985017600:0@lo:38:kworker.0 00000100:00080000:0.0:1767318843.661949:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.661952:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.661955:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985017856:0@lo:38:kworker.0 00000100:00100000:0.0:1767318844.123914:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985018240:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318844.123994:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318844.124147:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985018240 00000100:00100000:1.0:1767318844.124155:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985018240 00000100:00100000:1.0:1767318844.124243:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1972 00000100:00100000:1.0:1767318844.124261:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985018240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318844.124302:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318844.124355:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985018240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (361us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318844.124363:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1972 00000100:00100000:1.0:1767318844.124409:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985018240:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318844.125209:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985018368 00000100:00100000:0.0:1767318844.125218:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985018368 00000100:00100000:0.0:1767318844.125264:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1973 00000100:00100000:0.0:1767318844.125273:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985018368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318844.125292:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318844.125358:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985018368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (209us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318844.125365:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1973 00010000:00010000:1.0:1767318845.275458:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318846.299770:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318847.328992:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318848.347415:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318848.681917:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318848.682414:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4265 00000100:00100000:1.0:1767318848.682428:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090000a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920527616:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318848.682516:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090000a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920527616:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (609us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318848.682525:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4265 00000100:00080000:3.0:1767318848.682950:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318848.688078:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920527488 00000100:00100000:0.0:1767318848.688090:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920527488 00000100:00100000:0.0:1767318848.688143:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4623 00000100:00100000:0.0:1767318848.688152:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920527488:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767318848.688217:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318848.688587:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920527488:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 433us (964us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318848.688596:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4623 00000100:00100000:0.0:1767318848.732760:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985018752:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.732792:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318848.732825:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318848.732830:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985019008:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.732849:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318848.733304:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318848.733309:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318848.733696:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985018496:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.733744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318848.733784:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985018624:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.733809:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318848.733834:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985018880:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.733854:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318848.733872:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985019136:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.733891:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318848.733928:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985018624 00000100:00100000:1.0:1767318848.733935:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985018624 00000100:00100000:1.0:1767318848.733962:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1646 00000100:00100000:1.0:1767318848.733969:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985018624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318848.733987:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318848.734024:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985018624:12345-0@lo:38:kworker.0 Request processed in 54us (213us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318848.734030:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1646 00000100:00100000:0.0:1767318848.735900:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985018752 00000100:00100000:0.0:1767318848.735914:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985018752 00000100:00100000:0.0:1767318848.735972:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985019008 00000100:00100000:0.0:1767318848.735975:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985019008 00000100:00100000:0.0:1767318848.735990:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985018496 00000100:00100000:0.0:1767318848.735993:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985018496 00000100:00100000:0.0:1767318848.736003:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985018880 00000100:00100000:0.0:1767318848.736006:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985018880 00000100:00100000:0.0:1767318848.736014:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985019136 00000100:00100000:0.0:1767318848.736016:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985019136 00000100:00100000:0.0:1767318848.736027:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4266 00000100:00100000:0.0:1767318848.736049:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985018752:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.736133:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985018752:12345-0@lo:400:kworker.0 Request processed in 97us (3338us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318848.736139:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4266 00000100:00100000:0.0:1767318848.736157:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4267 00000100:00080000:1.0:1767318848.736628:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318848.736635:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.736639:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985018624:0@lo:38:kworker.0 00000100:00080000:3.0:1767318848.737754:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318848.741222:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318848.741235:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.741241:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985018496:0@lo:38:kworker.0 00000100:00080000:1.0:1767318848.741256:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318848.741259:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.741262:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985018880:0@lo:38:kworker.0 00000100:00080000:1.0:1767318848.741271:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318848.741274:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.741277:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985019136:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.741339:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985018752:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.755780:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985019008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.755991:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985019008:12345-0@lo:400:kworker.0 Request processed in 19826us (23138us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318848.756002:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4267 00000100:00100000:1.0:1767318848.756212:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985019008:0@lo:400:kworker.0 00000100:00100000:0.0:1767318849.243724:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985019520:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318849.243796:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318849.244497:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985019520 00000100:00100000:1.0:1767318849.244506:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985019520 00000100:00100000:1.0:1767318849.244575:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1974 00000100:00100000:1.0:1767318849.244585:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985019520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318849.244614:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318849.244687:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985019520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (890us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318849.244695:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1974 00000100:00100000:3.0:1767318849.246609:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985019648 00000100:00100000:3.0:1767318849.246619:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985019648 00000100:00100000:3.0:1767318849.246697:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1975 00000100:00100000:3.0:1767318849.246709:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985019648:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318849.246743:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318849.246816:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985019648:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (1148us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318849.246823:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1975 00010000:00010000:1.0:1767318851.419655:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318852.448932:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318853.468770:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318853.804807:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318853.804890:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318853.805034:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920527744 00000100:00100000:3.0:1767318853.805046:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920527744 00000100:00100000:1.0:1767318853.805096:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920527872 00000100:00100000:1.0:1767318853.805103:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920527872 00000100:00100000:1.0:1767318853.805204:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4268 00000100:00100000:1.0:1767318853.805214:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920527872:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318853.805283:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920527872:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (392us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318853.805291:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4268 00000100:00100000:1.0:1767318853.805309:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4627 00000100:00100000:1.0:1767318853.805314:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920527744:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318853.805333:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318853.805356:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920527744:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (559us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318853.805361:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4627 00000100:00100000:0.0:1767318853.851759:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318853.851870:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318853.851889:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985020288:0@lo:400:kworker.0 00000100:00100000:3.0:1767318853.851952:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318853.852024:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985020288 00000100:00100000:3.0:1767318853.852032:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985020288 00000100:00100000:3.0:1767318853.852091:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4269 00000100:00100000:3.0:1767318853.852099:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090003100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985020288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318853.852146:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090003100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985020288:12345-0@lo:400:kworker.0 Request processed in 48us (194us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318853.852154:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4269 00000100:00100000:3.0:1767318853.852221:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985020288:0@lo:400:kworker.0 00000100:00080000:3.0:1767318853.852251:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318853.853320:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985019776:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.853374:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318853.853436:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985019904:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.853472:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318853.853499:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985020160:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.853530:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318853.853580:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985019904 00000100:00100000:1.0:1767318853.853588:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985019904 00000100:00100000:1.0:1767318853.853620:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1647 00000100:00100000:1.0:1767318853.853626:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985019904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318853.853647:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318853.853685:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985019904:12345-0@lo:38:kworker.0 Request processed in 57us (211us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318853.853691:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1647 00000100:00080000:1.0:1767318853.853727:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318853.853732:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318853.853735:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985019904:0@lo:38:kworker.0 00000100:00100000:3.0:1767318853.854795:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985019776 00000100:00100000:3.0:1767318853.854809:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985019776 00000100:00100000:3.0:1767318853.854844:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985020160 00000100:00100000:3.0:1767318853.854847:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985020160 00000100:00100000:3.0:1767318853.854867:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4628 00000100:00100000:3.0:1767318853.854873:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985019776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318853.854888:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318853.854936:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985019776:12345-0@lo:38:kworker.0 Request processed in 62us (1562us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318853.854943:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4628 00000100:00100000:3.0:1767318853.854956:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4629 00000100:00100000:3.0:1767318853.854960:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985020160:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318853.854968:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318853.854989:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985020160:12345-0@lo:38:kworker.0 Request processed in 28us (1468us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318853.854994:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4629 00000100:00100000:0.0:1767318853.855094:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985020032:0@lo:400:kworker.0 00000100:00100000:0.0:1767318853.855260:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318853.857152:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318853.857222:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318853.857231:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985019776:0@lo:38:kworker.0 00000100:00080000:0.0:1767318853.857270:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318853.857274:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318853.857277:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985020160:0@lo:38:kworker.0 00000100:00100000:0.0:1767318853.857825:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985020416:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.860077:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985020032 00000100:00100000:1.0:1767318853.860094:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985020032 00000100:00100000:0.0:1767318853.860493:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318853.860711:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985020416 00000100:00100000:3.0:1767318853.860717:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985020416 00000100:00100000:0.0:1767318853.860750:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2822 00000100:00100000:0.0:1767318853.860762:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985020672:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318853.860829:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985020672:12345-0@lo:400:kworker.0 Request processed in 71us (2856us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318853.860836:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2822 00000100:00080000:0.0:1767318853.860862:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318853.860867:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318853.860869:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318853.861070:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4270 00000100:00100000:1.0:1767318853.861083:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985020032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318853.861130:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985020032:12345-0@lo:400:kworker.0 Request processed in 51us (5870us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318853.861620:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4630 00000100:00100000:3.0:1767318853.861626:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985020416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318853.861645:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318853.861688:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985020416:12345-0@lo:38:kworker.0 Request processed in 61us (2865us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318853.861695:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4630 00000100:00080000:3.0:1767318853.861729:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318853.861734:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318853.861737:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985020416:0@lo:38:kworker.0 00000100:00100000:3.0:1767318853.861775:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985020032:0@lo:400:kworker.0 00000100:00100000:1.0:1767318853.862763:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4270 00000100:00100000:0.0:1767318854.363736:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985020928:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318854.363814:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318854.363867:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318854.363874:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce090000000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985020800:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318854.363896:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318854.363954:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985020800 00000100:00100000:0.0:1767318854.363960:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985020800 00000100:00100000:0.0:1767318854.364018:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1977 00000100:00100000:0.0:1767318854.364031:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985020800:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318854.364059:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318854.364115:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985020800:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (217us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318854.364127:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1977 00000100:00100000:0.0:1767318854.364203:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce090000000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985020800:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318854.364219:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985020928:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767318854.491750:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318857.563410:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318858.590067:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318858.924615:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318858.924669:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318858.924955:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920528000 00000100:00100000:1.0:1767318858.924965:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920528000 00000100:00100000:1.0:1767318858.925010:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920528128 00000100:00100000:1.0:1767318858.925014:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920528128 00000100:00100000:1.0:1767318858.925065:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4271 00000100:00100000:1.0:1767318858.925074:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920528128:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318858.925160:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920528128:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 88us (490us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318858.925204:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4271 00000100:00100000:1.0:1767318858.925223:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4631 00000100:00100000:1.0:1767318858.925227:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce090000000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920528000:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318858.925247:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318858.925272:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce090000000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920528000:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (666us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318858.925278:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4631 00000100:00080000:3.0:1767318858.928637:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318858.972179:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.972190:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318858.972193:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.972197:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.972201:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.972209:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.972214:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.972223:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318858.972262:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985021056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318858.972278:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.972282:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.972283:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.972287:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.972289:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.972294:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318858.972308:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22d80 x1853163985021184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318858.972352:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985021056:0@lo:38:kworker.0 00000100:00080000:3.0:1767318858.972368:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.972373:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.972375:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.972379:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.972383:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.972386:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318858.972399:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6fd1880 x1853163985021440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318858.972430:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.972433:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.972435:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.972438:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.972441:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.972446:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318858.972458:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318858.972474:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6fd2680 x1853163985021696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318858.972560:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985021952:0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.972576:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985021184:0@lo:38:kworker.0 00000100:00100000:0.0:1767318858.972644:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318858.972644:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318858.972695:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318858.972708:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985021824:0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.972721:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985021440:0@lo:38:kworker.0 00000100:00100000:3.0:1767318858.972747:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.972764:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.972800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985021696:0@lo:38:kworker.0 00000100:00100000:3.0:1767318858.972802:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985021056 00000100:00100000:3.0:1767318858.972812:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985021056 00000100:00100000:0.0:1767318858.972833:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318858.973739:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985021568:0@lo:400:kworker.0 00000100:00100000:1.0:1767318858.973808:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985021184 00000100:00100000:1.0:1767318858.973813:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985021184 00000100:00100000:1.0:1767318858.973839:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1648 00000100:00100000:1.0:1767318858.973844:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985021184:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318858.973864:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318858.973903:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985021184:12345-0@lo:38:kworker.0 Request processed in 58us (1269us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318858.973909:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1648 00000100:00080000:1.0:1767318858.973955:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318858.973959:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318858.973963:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985021184:0@lo:38:kworker.0 00000100:00100000:3.0:1767318858.980602:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985021440 00000100:00100000:3.0:1767318858.980616:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985021440 00000100:00100000:3.0:1767318858.980671:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985021696 00000100:00100000:3.0:1767318858.980675:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985021696 00000100:00100000:3.0:1767318858.980683:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985021312 00000100:00100000:3.0:1767318858.980685:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985021312 00000100:00100000:3.0:1767318858.980719:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4272 00000100:00100000:3.0:1767318858.980728:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985021312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318858.982273:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985021312:12345-0@lo:400:kworker.0 Request processed in 1539us (9034us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318858.982286:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4272 00000100:00100000:3.0:1767318858.982321:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4632 00000100:00100000:3.0:1767318858.982328:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985021056:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318858.982348:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318858.982386:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985021056:12345-0@lo:38:kworker.0 Request processed in 57us (9931us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318858.982392:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4632 00000100:00100000:3.0:1767318858.982404:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4633 00000100:00100000:3.0:1767318858.982408:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985021440:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318858.982417:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318858.982440:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985021440:12345-0@lo:38:kworker.0 Request processed in 29us (9678us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318858.982446:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4633 00000100:00100000:3.0:1767318858.982455:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4634 00000100:00100000:3.0:1767318858.982473:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985021696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318858.982483:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318858.982505:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985021696:12345-0@lo:38:kworker.0 Request processed in 45us (9673us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318858.982510:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4634 00000100:00080000:3.0:1767318858.982538:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318858.982544:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318858.982547:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318858.982584:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985021824:0@lo:400:kworker.0 00000100:00100000:3.0:1767318858.982597:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985021952:0@lo:400:kworker.0 00000100:00100000:3.0:1767318858.982740:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985021312:0@lo:400:kworker.0 00000100:00080000:3.0:1767318858.983974:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318858.983983:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318858.983987:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985021440:0@lo:38:kworker.0 00000100:00080000:3.0:1767318858.983998:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318858.984001:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318858.984004:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985021696:0@lo:38:kworker.0 00000100:00080000:3.0:1767318858.984014:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318858.984017:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318858.984020:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985021056:0@lo:38:kworker.0 00000100:00100000:3.0:1767318858.988477:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985021568 00000100:00100000:3.0:1767318858.988492:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985021568 00000100:00100000:3.0:1767318858.988562:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4273 00000100:00100000:3.0:1767318858.988572:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985021568:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318858.988723:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318858.991494:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985021568:12345-0@lo:400:kworker.0 Request processed in 2914us (13111us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318858.991509:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4273 00000100:00100000:0.0:1767318859.483491:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318859.483727:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985022080:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318859.484134:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318859.484286:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985022208:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318859.484320:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318859.484921:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985022080:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318859.485187:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985022208:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318859.611366:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318860.636645:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318863.067977:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318838/real 1767318838] req@ffff8ce1b351a300 x1853163985015808/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318863 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318863.068006:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b351a300 x1853163985015808/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318863 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318863.068022:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318863.068053:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318863.068057:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985015808:0@lo:250:kworker.0 00000100:00100000:1.0:1767318863.068657:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985022720:0@lo:400:kworker.0 00000100:00100000:1.0:1767318863.068725:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318863.068786:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318863.068795:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985022976:0@lo:400:kworker.0 00000100:00100000:1.0:1767318863.068829:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318863.068899:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985023232 00000100:00100000:1.0:1767318863.068906:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985023232 00000100:00100000:1.0:1767318863.068959:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985023360 00000100:00100000:1.0:1767318863.068961:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985023360 00000100:00100000:1.0:1767318863.068980:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2825 00000100:00100000:1.0:1767318863.068987:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985023232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318863.069026:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2826 00000100:00100000:1.0:1767318863.069034:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985023232:12345-0@lo:400:kworker.0 Request processed in 46us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318863.069040:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985023360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318863.069046:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2825 00000100:00100000:3.0:1767318863.069086:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985023360:12345-0@lo:400:kworker.0 Request processed in 52us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318863.069089:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985022336:0@lo:250:kworker.0 00000100:00100000:3.0:1767318863.069098:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2826 00000400:00000100:1.0:1767318863.069117:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985022336 offset 0 length 520: 4 00000100:00100000:3.0:1767318863.069122:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985022720 00000100:00100000:3.0:1767318863.069127:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985022720 00000100:00100000:1.0:1767318863.069142:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985022464:0@lo:38:kworker.0 00000100:00100000:3.0:1767318863.069184:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985022976 00000100:00100000:3.0:1767318863.069188:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985022976 00000100:00100000:1.0:1767318863.069192:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318863.069213:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985022592:0@lo:38:kworker.0 00000100:00100000:3.0:1767318863.069215:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985022464 00000100:00100000:3.0:1767318863.069221:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985022464 00000100:00100000:3.0:1767318863.069246:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4274 00000100:00100000:1.0:1767318863.069247:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318863.069257:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985022720:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318863.069270:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985022848:0@lo:38:kworker.0 00000100:00100000:0.0:1767318863.069360:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4275 00000100:00100000:0.0:1767318863.069370:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985022976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318863.071545:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318863.071573:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985023104:0@lo:38:kworker.0 00000100:00100000:1.0:1767318863.071601:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318863.071636:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985022848 00000100:00100000:1.0:1767318863.071643:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985022848 00000100:00100000:1.0:1767318863.071663:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985023104 00000100:00100000:1.0:1767318863.071666:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985023104 00000100:00100000:1.0:1767318863.071677:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4635 00000100:00100000:1.0:1767318863.071684:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985022464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318863.071704:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318863.071748:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985022464:12345-0@lo:38:kworker.0 Request processed in 63us (2558us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318863.071755:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4635 00000100:00100000:1.0:1767318863.071772:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4636 00000100:00100000:1.0:1767318863.071776:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985022848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318863.071785:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318863.071809:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985022848:12345-0@lo:38:kworker.0 Request processed in 32us (268us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318863.071814:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4636 00000100:00100000:1.0:1767318863.071825:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4637 00000100:00100000:1.0:1767318863.071828:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985023104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318863.071836:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318863.071852:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985023104:12345-0@lo:38:kworker.0 Request processed in 22us (250us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318863.071856:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4637 00000100:00100000:1.0:1767318863.071873:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985022592 00000100:00100000:1.0:1767318863.071877:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985022592 00000100:00100000:1.0:1767318863.071895:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1649 00000100:00100000:1.0:1767318863.071898:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985022592:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318863.071906:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318863.071927:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985022592:12345-0@lo:38:kworker.0 Request processed in 28us (2685us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318863.071932:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1649 00000100:00080000:1.0:1767318863.071965:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318863.071969:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318863.071973:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985022464:0@lo:38:kworker.0 00000100:00080000:1.0:1767318863.071981:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318863.071984:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318863.071987:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985022592:0@lo:38:kworker.0 00000100:00080000:1.0:1767318863.071995:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318863.071997:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318863.072000:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985022848:0@lo:38:kworker.0 00000100:00080000:1.0:1767318863.072006:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318863.072009:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318863.072011:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985023104:0@lo:38:kworker.0 00000100:00100000:3.0:1767318863.072562:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985022720:12345-0@lo:400:kworker.0 Request processed in 3304us (3834us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318863.072570:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4274 00000100:00100000:1.0:1767318863.072669:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985022720:0@lo:400:kworker.0 00000100:00100000:0.0:1767318863.075955:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985022976:12345-0@lo:400:kworker.0 Request processed in 6578us (7117us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318863.075970:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4275 00000100:00100000:3.0:1767318863.076601:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985022976:0@lo:400:kworker.0 00010000:00010000:1.0:1767318863.708017:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318864.043596:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318864.043632:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767318864.043654:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920528512 offset 0 length 520: 4 00000100:00100000:0.0:1767318864.043658:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920528384 00000100:00100000:0.0:1767318864.043670:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920528384 00000100:00100000:0.0:1767318864.043743:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920528256 00000100:00100000:0.0:1767318864.043746:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920528256 00000100:00100000:0.0:1767318864.043767:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4276 00000100:00100000:0.0:1767318864.043776:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920528384:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767318864.043853:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920528384:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (264us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318864.043862:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4276 00000100:00100000:0.0:1767318864.043883:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4638 00000100:00100000:0.0:1767318864.043888:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920528256:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767318864.043907:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318864.043933:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920528256:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (301us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318864.043941:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4638 00000100:00100000:1.0:1767318864.604455:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318864.604494:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985023488:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318864.604626:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318864.606857:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985023488:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318864.607872:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318864.607899:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985023616:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318864.607968:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318864.608299:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1981 00000100:00100000:1.0:1767318864.608310:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985023616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318864.608349:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318864.608411:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985023616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (443us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318864.608418:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1981 00000100:00100000:0.0:1767318864.608547:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985023616:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318864.731418:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318865.755439:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318866.779836:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318868.189490:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318868.189501:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318868.189504:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318868.189509:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318868.189513:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318868.189522:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4111 00000100:00080000:0.0:1767318868.189526:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318868.189533:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318868.189574:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60700 x1853163985023744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318868.189611:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318868.189615:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318868.189617:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318868.189620:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4111 00000100:00080000:0.0:1767318868.189623:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318868.189626:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318868.189641:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163985023872/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318868.189670:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318868.189673:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318868.189675:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318868.189678:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4111 00000100:00080000:0.0:1767318868.189680:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318868.189684:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318868.189695:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163985024128/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318868.189710:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318868.189714:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318868.189715:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318868.189718:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4111 00000100:00080000:0.0:1767318868.189721:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318868.189724:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318868.189733:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63100 x1853163985024384/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318868.189801:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985024256:0@lo:400:kworker.0 00000100:00100000:1.0:1767318868.189906:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985024512:0@lo:400:kworker.0 00000100:00100000:3.0:1767318868.189977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985023744:0@lo:38:kworker.0 00000100:00100000:0.0:1767318868.190691:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985024640:0@lo:400:kworker.0 00000100:00100000:0.0:1767318868.195550:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318868.195734:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318868.196017:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985024640 00000100:00100000:0.0:1767318868.196028:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985024640 00000100:00100000:0.0:1767318868.196089:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985024512 00000100:00100000:0.0:1767318868.196093:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985024512 00000100:00100000:0.0:1767318868.196116:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2827 00000100:00100000:0.0:1767318868.196126:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985024640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318868.196208:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985024640:12345-0@lo:400:kworker.0 Request processed in 84us (666us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318868.196215:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2827 00000100:00100000:0.0:1767318868.196233:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2828 00000100:00100000:0.0:1767318868.196239:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985024512:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318868.196269:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985024512:12345-0@lo:400:kworker.0 Request processed in 30us (534us total) trans 0 rc 0/0 00000100:00080000:1.0:1767318868.196513:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318868.196518:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318868.196552:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985024512:0@lo:400:kworker.0 00000100:00100000:1.0:1767318868.196605:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985024640:0@lo:400:kworker.0 00000100:00100000:0.0:1767318868.196647:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2828 00000100:00100000:3.0:1767318868.196746:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318868.196968:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985023872:0@lo:38:kworker.0 00000100:00100000:3.0:1767318868.196997:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318868.197025:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985024128:0@lo:38:kworker.0 00000100:00100000:3.0:1767318868.197049:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318868.200566:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985023872 00000100:00100000:0.0:1767318868.200585:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985023872 00000100:00100000:0.0:1767318868.200659:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1650 00000100:00100000:0.0:1767318868.200668:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985023872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318868.200694:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318868.200771:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985023872:12345-0@lo:38:kworker.0 Request processed in 102us (3772us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318868.200778:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1650 00000100:00100000:3.0:1767318868.203256:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985024384:0@lo:38:kworker.0 00000100:00100000:3.0:1767318868.203349:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318868.203628:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985023744 00000100:00100000:1.0:1767318868.203636:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985023744 00000100:00100000:1.0:1767318868.203723:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985024128 00000100:00100000:1.0:1767318868.203727:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985024128 00000100:00100000:0.0:1767318868.203767:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4639 00000100:00100000:0.0:1767318868.203776:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985023744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318868.203812:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318868.212593:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985023744:12345-0@lo:38:kworker.0 Request processed in 8803us (15834us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318868.212608:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4639 00000100:00080000:3.0:1767318868.215598:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318868.215611:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318868.215616:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985023744:0@lo:38:kworker.0 00000100:00080000:3.0:1767318868.215632:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318868.215635:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318868.215638:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985023872:0@lo:38:kworker.0 00000100:00080000:3.0:1767318868.215648:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318868.215653:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318868.215656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985024128:0@lo:38:kworker.0 00000100:00100000:0.0:1767318868.215818:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985024384 00000100:00100000:0.0:1767318868.215836:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985024384 00000100:00100000:0.0:1767318868.215883:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4641 00000100:00100000:0.0:1767318868.215890:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985024384:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318868.215914:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318868.215981:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985024384:12345-0@lo:38:kworker.0 Request processed in 90us (12632us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318868.215989:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4641 00000100:00080000:3.0:1767318868.216046:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318868.216050:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318868.216053:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985024384:0@lo:38:kworker.0 00000100:00100000:3.0:1767318869.162970:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318869.163025:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318869.163102:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920528768 00000100:00100000:1.0:1767318869.163111:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920528768 00000100:00100000:1.0:1767318869.163217:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920528640 00000100:00100000:1.0:1767318869.163221:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920528640 00000100:00100000:1.0:1767318869.163240:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4279 00000100:00100000:1.0:1767318869.163250:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920528768:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318869.163327:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920528768:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (366us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318869.163338:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4279 00000100:00100000:1.0:1767318869.163356:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4642 00000100:00100000:1.0:1767318869.163361:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920528640:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318869.163379:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318869.163405:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920528640:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (380us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318869.163411:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4642 00000100:00100000:3.0:1767318869.724820:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985024768 00000100:00100000:3.0:1767318869.724829:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985024768 00000100:00100000:3.0:1767318869.724884:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1982 00000100:00100000:3.0:1767318869.724902:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985024768:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318869.724932:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318869.724986:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985024768:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (341us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318869.724995:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1982 00000100:00100000:0.0:1767318869.725082:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985024768:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318869.725419:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985024896:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318869.725450:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318869.725710:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985024896:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318869.851627:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318870.875931:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318871.899649:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318872.923497:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318873.307357:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318873.307375:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318873.307379:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318873.307383:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318873.307388:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318873.307397:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4116 00000100:00080000:0.0:1767318873.307402:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318873.307410:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318873.307450:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63100 x1853163985025024/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318873.307483:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318873.307486:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318873.307488:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318873.307492:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4116 00000100:00080000:0.0:1767318873.307495:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318873.307498:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318873.307511:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163985025152/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318873.307554:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318873.307557:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318873.307559:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318873.307562:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4116 00000100:00080000:0.0:1767318873.307572:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318873.307575:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318873.307588:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60700 x1853163985025408/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318873.307604:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318873.307607:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318873.307608:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318873.307611:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4116 00000100:00080000:0.0:1767318873.307613:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318873.307616:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318873.307629:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63800 x1853163985025664/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318873.307696:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985025920:0@lo:400:kworker.0 00000100:00100000:0.0:1767318873.307758:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318873.307826:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318873.307832:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985025792:0@lo:400:kworker.0 00000100:00100000:0.0:1767318873.307854:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318873.307885:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025920 00000100:00100000:0.0:1767318873.307892:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025920 00000100:00100000:3.0:1767318873.308513:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985025024:0@lo:38:kworker.0 00000100:00100000:3.0:1767318873.308545:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318873.308568:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985025152:0@lo:38:kworker.0 00000100:00100000:3.0:1767318873.308592:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318873.308616:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985025408:0@lo:38:kworker.0 00000100:00100000:3.0:1767318873.308637:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318873.308655:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985025664:0@lo:38:kworker.0 00000100:00100000:3.0:1767318873.308672:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318873.308699:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025152 00000100:00100000:3.0:1767318873.308704:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025152 00000100:00100000:3.0:1767318873.308732:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1651 00000100:00100000:3.0:1767318873.308738:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985025152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318873.308759:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318873.308800:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985025152:12345-0@lo:38:kworker.0 Request processed in 60us (206us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318873.308807:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1651 00000100:00100000:3.0:1767318873.308831:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025536 00000100:00100000:3.0:1767318873.308835:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025536 00000100:00100000:3.0:1767318873.308863:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025280 00000100:00100000:3.0:1767318873.308866:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025280 00000100:00100000:3.0:1767318873.308880:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025024 00000100:00100000:3.0:1767318873.308882:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025024 00000100:00100000:3.0:1767318873.308889:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025408 00000100:00100000:3.0:1767318873.308891:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025408 00000100:00100000:3.0:1767318873.308898:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025664 00000100:00100000:3.0:1767318873.308900:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025664 00000100:00100000:3.0:1767318873.308908:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4280 00000100:00100000:3.0:1767318873.308913:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985025536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318873.308969:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985025536:12345-0@lo:400:kworker.0 Request processed in 55us (638us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318873.308976:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4280 00000100:00100000:3.0:1767318873.308989:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4281 00000100:00100000:3.0:1767318873.308995:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985025280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318873.309021:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985025280:12345-0@lo:400:kworker.0 Request processed in 27us (598us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318873.309026:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4281 00000100:00100000:3.0:1767318873.309038:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4643 00000100:00100000:3.0:1767318873.309042:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6fd0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985025024:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318873.309052:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318873.309117:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6fd0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985025024:12345-0@lo:38:kworker.0 Request processed in 73us (571us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318873.309123:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4643 00000100:00100000:3.0:1767318873.309134:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4644 00000100:00100000:3.0:1767318873.309138:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985025408:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318873.309146:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318873.309208:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985025408:12345-0@lo:38:kworker.0 Request processed in 68us (569us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318873.309213:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4644 00000100:00100000:3.0:1767318873.309225:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4645 00000100:00100000:3.0:1767318873.309228:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985025664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318873.309236:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318873.309258:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985025664:12345-0@lo:38:kworker.0 Request processed in 28us (585us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318873.309263:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4645 00000100:00100000:3.0:1767318873.309303:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985025280:0@lo:400:kworker.0 00000100:00100000:3.0:1767318873.309314:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985025536:0@lo:400:kworker.0 00000100:00080000:3.0:1767318873.309344:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318873.309349:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318873.309352:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985025024:0@lo:38:kworker.0 00000100:00080000:3.0:1767318873.309358:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318873.309361:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318873.309364:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985025152:0@lo:38:kworker.0 00000100:00080000:3.0:1767318873.309371:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318873.309374:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318873.309377:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985025408:0@lo:38:kworker.0 00000100:00080000:3.0:1767318873.309384:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318873.309387:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318873.309389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985025664:0@lo:38:kworker.0 00000100:00080000:3.0:1767318873.309432:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318873.309437:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318873.309792:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985025792 00000100:00100000:0.0:1767318873.309798:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985025792 00000100:00080000:3.0:1767318873.311216:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318873.328373:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985025920:0@lo:400:kworker.0 00000100:00100000:3.0:1767318873.333432:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985025792:0@lo:400:kworker.0 00000100:00100000:1.0:1767318874.283084:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318874.284699:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318874.288155:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920529024 00000100:00100000:0.0:1767318874.288643:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920529024 00000100:00100000:0.0:1767318874.288799:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920528896 00000100:00100000:0.0:1767318874.288803:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920528896 00000100:00100000:0.0:1767318874.288830:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4282 00000100:00100000:0.0:1767318874.288840:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920529024:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767318874.289003:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920529024:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 164us (5928us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318874.289015:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4282 00000100:00100000:0.0:1767318874.289035:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4646 00000100:00100000:0.0:1767318874.289040:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920528896:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767318874.289059:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318874.289088:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920528896:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (4396us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318874.289095:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4646 00000100:00100000:0.0:1767318874.843495:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318874.843520:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985026048:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318874.843589:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318874.843723:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026048 00000100:00100000:1.0:1767318874.843732:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026048 00000100:00100000:1.0:1767318874.843788:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1984 00000100:00100000:1.0:1767318874.843797:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985026048:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318874.843826:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318874.843884:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985026048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (293us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318874.843891:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1984 00000100:00100000:0.0:1767318874.843986:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985026048:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318874.844475:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026176 00000100:00100000:0.0:1767318874.844481:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026176 00000100:00100000:0.0:1767318874.844515:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1985 00000100:00100000:0.0:1767318874.844522:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985026176:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318874.844539:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318874.844583:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985026176:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (188us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318874.844589:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1985 00000100:00100000:3.0:1767318874.844785:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985026176:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318875.995314:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318877.019599:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318878.043676:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318878.430437:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318878.430447:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318878.430451:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318878.430455:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318878.430459:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318878.430467:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4121 00000100:00080000:0.0:1767318878.430472:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318878.430478:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318878.430515:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5500 x1853163985026304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318878.430543:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318878.430547:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318878.430549:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318878.430553:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4121 00000100:00080000:0.0:1767318878.430556:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318878.430560:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318878.430575:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985026432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318878.430618:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318878.430621:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318878.430623:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318878.430628:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4121 00000100:00080000:0.0:1767318878.430631:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318878.430634:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318878.430648:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985026688/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318878.430658:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:0.0:1767318878.430665:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318878.430669:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318878.430671:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318878.430675:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4121 00000100:00080000:0.0:1767318878.430677:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767318878.430681:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985026816:0@lo:400:kworker.0 00000100:00080000:0.0:1767318878.430683:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318878.430697:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88700 x1853163985026944/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318878.430754:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318878.430756:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985027072:0@lo:400:kworker.0 00000100:00100000:0.0:1767318878.430804:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318878.430804:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985026560:0@lo:400:kworker.0 00000100:00100000:3.0:1767318878.430864:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985026304:0@lo:38:kworker.0 00000100:00100000:1.0:1767318878.431074:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318878.431411:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026816 00000100:00100000:1.0:1767318878.431417:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026816 00000100:00100000:1.0:1767318878.431519:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026560 00000100:00100000:1.0:1767318878.431523:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026560 00000100:00100000:1.0:1767318878.431545:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4283 00000100:00100000:1.0:1767318878.431551:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985026816:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318878.431586:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985026816:12345-0@lo:400:kworker.0 Request processed in 34us (834us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318878.431593:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4283 00000100:00100000:1.0:1767318878.431605:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4284 00000100:00100000:1.0:1767318878.431610:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985026560:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318878.431629:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985026560:12345-0@lo:400:kworker.0 Request processed in 20us (557us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318878.431634:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4284 00000100:00100000:1.0:1767318878.431665:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985026816:0@lo:400:kworker.0 00000100:00100000:1.0:1767318878.431674:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985026560:0@lo:400:kworker.0 00000100:00100000:1.0:1767318878.431702:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318878.431724:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985027200 00000100:00100000:1.0:1767318878.431727:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985027200 00000100:00100000:1.0:1767318878.431747:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2832 00000100:00100000:1.0:1767318878.431752:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985027200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318878.431778:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985027200:12345-0@lo:400:kworker.0 Request processed in 26us (76us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318878.431784:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2832 00000100:00100000:1.0:1767318878.431804:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985027200:0@lo:400:kworker.0 00000100:00080000:1.0:1767318878.431830:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318878.431834:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318878.431836:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318878.432562:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318878.432590:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985026432:0@lo:38:kworker.0 00000100:00100000:3.0:1767318878.432615:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318878.432630:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026304 00000100:00100000:1.0:1767318878.432636:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026304 00000100:00100000:3.0:1767318878.432637:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985026688:0@lo:38:kworker.0 00000100:00100000:3.0:1767318878.432656:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318878.432658:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4647 00000100:00100000:1.0:1767318878.432663:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985026304:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318878.432671:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985026944:0@lo:38:kworker.0 00010000:00020000:1.0:1767318878.432680:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318878.432687:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318878.432714:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318878.432719:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318878.432723:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985026304:0@lo:38:kworker.0 00000100:00100000:3.0:1767318878.432745:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026432 00000100:00100000:3.0:1767318878.432748:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026432 00000100:00100000:3.0:1767318878.432766:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1652 00000100:00100000:3.0:1767318878.432770:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985026432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318878.432779:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318878.432804:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985026432:12345-0@lo:38:kworker.0 Request processed in 32us (187us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318878.432810:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1652 00000100:00100000:3.0:1767318878.432828:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026688 00000100:00100000:3.0:1767318878.432831:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026688 00000100:00100000:3.0:1767318878.432842:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985026944 00000100:00100000:3.0:1767318878.432844:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985026944 00000100:00100000:3.0:1767318878.432854:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4648 00000100:00100000:3.0:1767318878.432858:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985026688:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318878.432866:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318878.432885:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985026688:12345-0@lo:38:kworker.0 Request processed in 26us (228us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318878.432890:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4648 00000100:00100000:3.0:1767318878.432989:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4649 00000100:00100000:3.0:1767318878.432994:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985026944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318878.433003:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318878.433024:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985026944:12345-0@lo:38:kworker.0 Request processed in 29us (336us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318878.433029:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4649 00000100:00080000:3.0:1767318878.433058:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318878.433329:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318878.433333:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985026432:0@lo:38:kworker.0 00000100:00080000:3.0:1767318878.433345:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318878.433348:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318878.433351:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985026688:0@lo:38:kworker.0 00000100:00080000:3.0:1767318878.433357:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318878.433360:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318878.433363:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985026944:0@lo:38:kworker.0 00000100:00100000:1.0:1767318878.435681:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985026304:12345-0@lo:38:kworker.0 Request processed in 3005us (3109us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318878.435694:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4647 00000100:00100000:0.0:1767318878.437627:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985027072:0@lo:400:kworker.0 00010000:00010000:0.0:1767318879.067598:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318879.404055:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318879.404244:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318879.407854:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920529280 00000100:00100000:1.0:1767318879.407869:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920529280 00000100:00080000:3.0:1767318879.407963:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318879.963428:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318879.963447:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985027328:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318879.963517:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318879.963566:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985027456:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318879.963589:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318879.963653:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985027328 00000100:00100000:3.0:1767318879.963662:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985027328 00000100:00100000:3.0:1767318879.963719:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985027456 00000100:00100000:3.0:1767318879.963722:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985027456 00000100:00100000:3.0:1767318879.963747:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1986 00000100:00100000:3.0:1767318879.963756:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985027328:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318879.963790:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318879.963836:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985027328:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (318us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318879.963844:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1986 00000100:00100000:3.0:1767318879.963860:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1987 00000100:00100000:3.0:1767318879.963867:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985027456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318879.963881:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318879.963916:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985027456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (326us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318879.963922:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1987 00000100:00100000:1.0:1767318879.964053:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985027456:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318879.964076:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985027328:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318882.139882:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318883.163494:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318883.547265:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318883.547276:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318883.547279:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318883.547282:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318883.547286:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318883.547295:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4126 00000100:00080000:0.0:1767318883.547299:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318883.547306:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318883.547346:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985027584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318883.547381:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318883.547385:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318883.547387:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318883.547391:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4126 00000100:00080000:0.0:1767318883.547393:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318883.547397:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318883.547410:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a300 x1853163985027712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318883.547442:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985027584:0@lo:38:kworker.0 00000100:00080000:0.0:1767318883.547453:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318883.547457:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318883.547459:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318883.547462:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4126 00000100:00080000:0.0:1767318883.547465:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318883.547468:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318883.547479:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88700 x1853163985027968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318883.547495:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318883.547498:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318883.547499:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318883.547502:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4126 00000100:00100000:3.0:1767318883.547509:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318883.547514:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318883.547517:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318883.547527:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593b800 x1853163985028224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318883.547590:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985027840:0@lo:400:kworker.0 00000100:00100000:1.0:1767318883.547621:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318883.547718:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985027712:0@lo:38:kworker.0 00000100:00100000:3.0:1767318883.547743:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318883.551033:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985027584 00000100:00100000:0.0:1767318883.551048:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985027584 00000100:00100000:0.0:1767318883.551097:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985027840 00000100:00100000:0.0:1767318883.551100:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985027840 00000100:00100000:0.0:1767318883.551136:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985028096 00000100:00100000:0.0:1767318883.551139:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985028096 00000100:00100000:0.0:1767318883.551182:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4286 00000100:00100000:0.0:1767318883.551193:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985027840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318883.551270:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985027840:12345-0@lo:400:kworker.0 Request processed in 80us (3648us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318883.551277:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4286 00000100:00100000:0.0:1767318883.551294:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4287 00000100:00100000:0.0:1767318883.551299:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985028096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318883.551321:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985028096:12345-0@lo:400:kworker.0 Request processed in 22us (3578us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318883.551326:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4287 00000100:00100000:0.0:1767318883.551335:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4651 00000100:00100000:0.0:1767318883.551352:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985027584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318883.551374:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318883.551403:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985027584:12345-0@lo:38:kworker.0 Request processed in 50us (3894us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318883.551409:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4651 00000100:00080000:0.0:1767318883.551436:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318883.551548:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985028480:0@lo:400:kworker.0 00000100:00100000:0.0:1767318883.551585:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318883.554154:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985027712 00000100:00100000:3.0:1767318883.554160:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985027712 00000100:00100000:3.0:1767318883.554197:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1653 00000100:00100000:3.0:1767318883.554216:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985027712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318883.554237:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318883.557539:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318883.557563:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985028352:0@lo:400:kworker.0 00000100:00100000:3.0:1767318883.563503:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985027712:12345-0@lo:38:kworker.0 Request processed in 9291us (15752us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318883.563516:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1653 00000100:00100000:1.0:1767318883.565772:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985027840:0@lo:400:kworker.0 00000100:00100000:0.0:1767318883.568996:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318883.570289:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985028480:0@lo:400:kworker.0 00000100:00100000:3.0:1767318883.573206:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985028352:0@lo:400:kworker.0 00000100:00080000:0.0:1767318883.575143:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318884.187517:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318884.525483:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318884.525536:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:0.0:1767318884.526059:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318885.083884:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985028608 00000100:00100000:0.0:1767318885.083893:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985028608 00000100:00100000:0.0:1767318885.083958:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1988 00000100:00100000:0.0:1767318885.083967:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985028608:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318885.084024:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318885.084068:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985028608:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (276us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318885.084076:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1988 00000100:00100000:0.0:1767318885.084583:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985028736 00000100:00100000:0.0:1767318885.084586:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985028736 00000100:00100000:0.0:1767318885.084619:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1989 00000100:00100000:0.0:1767318885.084624:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985028736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318885.084640:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318885.084665:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985028736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (148us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318885.084671:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1989 00010000:00010000:0.0:1767318885.212515:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318888.283392:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767318888.627362:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920529664 offset 0 length 520: 4 00000100:00100000:0.0:1767318888.627406:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318888.627449:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318888.627542:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920529920 00000100:00100000:1.0:1767318888.627552:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920529920 00000100:00100000:1.0:1767318888.627626:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920529792 00000100:00100000:1.0:1767318888.627630:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920529792 00000100:00100000:1.0:1767318888.627648:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4289 00000100:00100000:1.0:1767318888.627657:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920529920:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318888.627733:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920529920:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (326us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318888.627740:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4289 00000100:00100000:1.0:1767318888.627758:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4655 00000100:00100000:1.0:1767318888.627762:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920529792:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318888.627782:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318888.627806:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920529792:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (357us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318888.627812:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4655 00000100:00080000:0.0:1767318888.627952:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318888.667469:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318863/real 1767318863] req@ffff8ce1a61c5180 x1853163985022336/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318888 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318888.667503:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61c5180 x1853163985022336/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318888 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318888.667524:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318888.667623:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318888.667628:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985022336:0@lo:250:kworker.0 00000100:00080000:1.0:1767318888.668488:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318888.668496:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318888.668499:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318888.668504:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4111 00000100:00080000:1.0:1767318888.668508:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318888.668514:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318888.668546:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985028864/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318888.668570:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318888.668575:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318888.668576:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318888.668580:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4131 00000100:00080000:1.0:1767318888.668583:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318888.668587:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318888.668604:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75500 x1853163985028992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318888.668607:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985028864:0@lo:250:kworker.0 00000100:00080000:1.0:1767318888.668621:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318888.668625:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318888.668626:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318888.668630:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4131 00000100:00080000:1.0:1767318888.668633:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318888.668636:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318888.668648:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985029120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:3.0:1767318888.668667:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985028864 offset 0 length 520: 4 00000100:00080000:1.0:1767318888.668692:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318888.668696:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318888.668699:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318888.668703:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4131 00000100:00080000:1.0:1767318888.668706:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318888.668709:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318888.668717:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985028992:0@lo:38:kworker.0 00000100:00100000:1.0:1767318888.668722:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985029376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318888.668740:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318888.668744:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318888.668745:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318888.668748:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4131 00000100:00080000:1.0:1767318888.668750:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318888.668753:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318888.668760:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d880 x1853163985029632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318888.668800:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985029760:0@lo:400:kworker.0 00000100:00100000:0.0:1767318888.668870:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985029504:0@lo:400:kworker.0 00000100:00100000:3.0:1767318888.671569:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.671579:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.676555:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318888.676579:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985029888:0@lo:400:kworker.0 00000100:00100000:1.0:1767318888.676635:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318888.676649:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985029120:0@lo:38:kworker.0 00000100:00100000:3.0:1767318888.676685:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.676697:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985029760 00000100:00100000:1.0:1767318888.676707:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985029760 00000100:00100000:3.0:1767318888.676717:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985029120 00000100:00100000:3.0:1767318888.676723:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985029120 00000100:00100000:1.0:1767318888.676764:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985029888 00000100:00100000:1.0:1767318888.676769:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985029888 00000100:00100000:3.0:1767318888.676773:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1654 00000100:00100000:3.0:1767318888.676781:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985029120:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318888.676799:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2835 00010000:00020000:3.0:1767318888.676805:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318888.676810:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985029760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318888.676860:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985029120:12345-0@lo:38:kworker.0 Request processed in 77us (172us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318888.676867:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1654 00000100:00100000:3.0:1767318888.676898:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985028992 00000100:00100000:3.0:1767318888.676912:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985028992 00000100:00100000:3.0:1767318888.676948:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4656 00000100:00100000:3.0:1767318888.676954:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985028992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318888.676966:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318888.677002:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985028992:12345-0@lo:38:kworker.0 Request processed in 46us (5439us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318888.677009:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4656 00000100:00080000:3.0:1767318888.677057:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318888.677065:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318888.677070:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985029120:0@lo:38:kworker.0 00000100:00100000:3.0:1767318888.677082:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985029248 00000100:00100000:3.0:1767318888.677086:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985029248 00000100:00100000:3.0:1767318888.677114:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4290 00000100:00100000:3.0:1767318888.677120:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985029248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318888.677183:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985029248:12345-0@lo:400:kworker.0 Request processed in 62us (151us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318888.677205:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4290 00000100:00100000:3.0:1767318888.677243:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985029376:0@lo:38:kworker.0 00000100:00100000:3.0:1767318888.677270:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318888.677293:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985029376 00000100:00100000:3.0:1767318888.677297:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985029376 00000100:00100000:3.0:1767318888.677323:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4657 00000100:00100000:3.0:1767318888.677331:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985029376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318888.677344:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318888.677382:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985029376:12345-0@lo:38:kworker.0 Request processed in 49us (111us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318888.677390:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4657 00000100:00100000:3.0:1767318888.677406:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4658 00000100:00100000:3.0:1767318888.677410:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985029632:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318888.677419:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318888.677447:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985029632:12345-0@lo:38:kworker.0 Request processed in 33us (92us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318888.677455:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4658 00000100:00100000:3.0:1767318888.677512:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985029888:0@lo:400:kworker.0 00000100:00100000:0.0:1767318888.681200:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318888.682976:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985029504 00000100:00100000:3.0:1767318888.682990:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985029504 00000100:00100000:3.0:1767318888.683070:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4291 00000100:00100000:3.0:1767318888.683079:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985029504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318888.683183:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985029504:12345-0@lo:400:kworker.0 Request processed in 87us (1973us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318888.683190:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4291 00000100:00100000:0.0:1767318888.684398:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985029504:0@lo:400:kworker.0 00000100:00100000:1.0:1767318888.684706:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985029760:12345-0@lo:400:kworker.0 Request processed in 7897us (13127us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318888.684719:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2835 00010000:00010000:1.0:1767318889.308006:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318890.203832:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985030144:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318890.203939:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318890.203988:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318890.203994:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985030016:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318890.204020:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318890.204112:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030144 00000100:00100000:3.0:1767318890.204120:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030144 00000100:00100000:3.0:1767318890.204211:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030016 00000100:00100000:3.0:1767318890.204215:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030016 00000100:00100000:3.0:1767318890.204240:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1990 00000100:00100000:3.0:1767318890.204248:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985030144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318890.204277:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318890.204320:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985030144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (380us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318890.204326:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1990 00000100:00100000:3.0:1767318890.204343:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1991 00000100:00100000:3.0:1767318890.204349:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985030016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318890.204363:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318890.204388:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985030016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 39us (366us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318890.204394:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1991 00000100:00080000:1.0:1767318890.204956:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318890.204961:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318890.204998:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985030016:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318890.205013:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985030144:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318890.333008:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318891.355374:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318893.740350:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318893.740394:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318893.740483:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920530048 00000100:00100000:3.0:1767318893.740503:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920530048 00000100:00100000:3.0:1767318893.740548:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920530176 00000100:00100000:3.0:1767318893.740551:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920530176 00000100:00100000:3.0:1767318893.740613:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4292 00000100:00100000:3.0:1767318893.740625:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920530176:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318893.740700:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920530176:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (304us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318893.740708:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4292 00000100:00100000:3.0:1767318893.740727:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4659 00000100:00100000:3.0:1767318893.740732:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920530048:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318893.740753:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318893.740781:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920530048:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (440us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318893.740787:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4659 00000100:00080000:3.0:1767318893.787828:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318893.787837:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318893.787840:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318893.787844:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318893.787848:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318893.787855:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4136 00000100:00080000:3.0:1767318893.787860:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318893.787865:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318893.787903:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985030272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318893.788223:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030272 00000100:00100000:1.0:1767318893.788231:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030272 00000100:00100000:1.0:1767318893.788264:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4660 00000100:00100000:1.0:1767318893.788271:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985030272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318893.788290:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318893.788649:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318893.788653:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318893.788656:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318893.788660:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4136 00000100:00080000:3.0:1767318893.788662:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318893.788666:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318893.788683:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9180 x1853163985030400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318893.788862:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030400 00000100:00100000:0.0:1767318893.788867:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030400 00000100:00100000:0.0:1767318893.788893:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1655 00000100:00100000:0.0:1767318893.788898:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985030400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318893.788908:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318893.788937:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985030400:12345-0@lo:38:kworker.0 Request processed in 38us (151us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318893.788944:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1655 00000100:00100000:1.0:1767318893.789521:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985030272:12345-0@lo:38:kworker.0 Request processed in 1247us (1491us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318893.789528:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4660 00000100:00080000:3.0:1767318893.792882:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318893.792890:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318893.792894:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318893.792902:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4136 00000100:00080000:3.0:1767318893.792907:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318893.792913:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318893.792936:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63800 x1853163985030656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318893.793623:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318893.793628:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318893.793630:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318893.793635:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4136 00000100:00080000:3.0:1767318893.793638:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318893.793642:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318893.793659:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163985030912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318893.793733:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985031040:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.793883:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985030528:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.793940:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318893.794079:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318893.794085:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985030784:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.794110:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318893.794140:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985031168:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.794184:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318893.794213:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985031168 00000100:00100000:1.0:1767318893.794219:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985031168 00000100:00100000:1.0:1767318893.794278:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2837 00000100:00100000:1.0:1767318893.794286:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985031168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.794332:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985031168:12345-0@lo:400:kworker.0 Request processed in 46us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318893.794339:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2837 00000100:00100000:0.0:1767318893.794529:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030528 00000100:00100000:0.0:1767318893.794534:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030528 00000100:00100000:0.0:1767318893.794564:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030784 00000100:00100000:0.0:1767318893.794567:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030784 00000100:00100000:0.0:1767318893.794588:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4293 00000100:00100000:0.0:1767318893.794594:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985030528:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318893.794633:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985030528:12345-0@lo:400:kworker.0 Request processed in 39us (692us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318893.794638:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4293 00000100:00100000:0.0:1767318893.794651:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4294 00000100:00100000:0.0:1767318893.794657:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985030784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318893.794681:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985030784:12345-0@lo:400:kworker.0 Request processed in 24us (571us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318893.794686:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4294 00000100:00100000:0.0:1767318893.794728:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985030784:0@lo:400:kworker.0 00000100:00100000:0.0:1767318893.794738:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985030528:0@lo:400:kworker.0 00000100:00100000:3.0:1767318893.794948:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318893.795585:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985031040 00000100:00100000:3.0:1767318893.795591:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985031040 00000100:00100000:3.0:1767318893.795624:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2838 00000100:00100000:3.0:1767318893.795631:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985031040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318893.795667:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985031040:12345-0@lo:400:kworker.0 Request processed in 37us (720us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318893.795673:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2838 00000100:00100000:3.0:1767318893.795709:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985031040:0@lo:400:kworker.0 00000100:00080000:3.0:1767318893.795741:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318893.795745:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318893.795747:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318893.795868:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985031168:0@lo:400:kworker.0 00000100:00100000:0.0:1767318893.796362:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030656 00000100:00100000:0.0:1767318893.796367:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030656 00000100:00100000:0.0:1767318893.796664:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985030912 00000100:00100000:0.0:1767318893.796668:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985030912 00000100:00100000:0.0:1767318893.796690:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4661 00000100:00100000:0.0:1767318893.796696:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985030656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318893.796715:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318893.797316:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318893.797322:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318893.797326:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985030656:0@lo:38:kworker.0 00000100:00080000:3.0:1767318893.797791:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318893.797795:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318893.797798:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985030912:0@lo:38:kworker.0 00000100:00100000:0.0:1767318893.798970:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985030656:12345-0@lo:38:kworker.0 Request processed in 2266us (2754us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318893.798980:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4661 00010000:00010000:1.0:1767318894.427499:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318895.323557:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318895.323582:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e376d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985031296:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318895.323651:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318895.323747:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985031296 00000100:00100000:1.0:1767318895.323756:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985031296 00000100:00100000:1.0:1767318895.323832:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1992 00000100:00100000:1.0:1767318895.323843:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985031296:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318895.323875:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318895.323924:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985031296:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (272us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318895.323932:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1992 00000100:00100000:1.0:1767318895.323978:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e376d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985031296:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318895.324038:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318895.325108:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985031424 00000100:00100000:3.0:1767318895.325114:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985031424 00000100:00100000:3.0:1767318895.325201:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1993 00000100:00100000:3.0:1767318895.325211:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985031424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318895.325231:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318895.325268:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985031424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 59us (239us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318895.325276:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1993 00010000:00010000:1.0:1767318895.451410:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318896.479475:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318897.499947:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318898.859024:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318898.859075:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318898.859197:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920530304 00000100:00100000:0.0:1767318898.859207:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920530304 00000100:00100000:0.0:1767318898.859251:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4295 00000100:00100000:0.0:1767318898.859263:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920530432:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767318898.859361:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318898.862908:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920530432:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 3623us (3808us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318898.862924:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4295 00000100:00080000:3.0:1767318898.907595:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.907607:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318898.907610:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.907615:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.907620:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.907629:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.907633:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.907641:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318898.907689:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63800 x1853163985031552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318898.907706:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.907709:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.907736:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.907741:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.907743:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.907748:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318898.907762:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63100 x1853163985031680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318898.908021:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767318898.908062:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985031808:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.908287:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318898.908877:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.908880:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.908885:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.908888:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.908893:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318898.908914:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62d80 x1853163985031936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318898.908952:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.908957:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.908959:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.908963:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.908965:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.908969:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318898.908982:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6680 x1853163985032192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318898.909003:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318898.909025:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985032064:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.909066:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.909214:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985032320:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.909267:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.909300:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318898.909521:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985032320 00000100:00100000:1.0:1767318898.909526:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985032320 00000100:00100000:0.0:1767318898.909559:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985032448:0@lo:400:kworker.0 00000100:00100000:1.0:1767318898.909572:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2839 00000100:00100000:1.0:1767318898.909585:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985032320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.909606:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318898.909650:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985032320:12345-0@lo:400:kworker.0 Request processed in 68us (381us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318898.909662:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2839 00000100:00100000:0.0:1767318898.909674:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985032320:0@lo:400:kworker.0 00000100:00100000:1.0:1767318898.909676:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985032448 00000100:00100000:1.0:1767318898.909684:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985032448 00000100:00100000:1.0:1767318898.909742:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2840 00000100:00100000:0.0:1767318898.909744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318898.909756:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985032448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.909789:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985031680:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.909821:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.909849:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985031936:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.909877:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318898.909896:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318898.909903:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985032192:0@lo:38:kworker.0 00000100:00080000:3.0:1767318898.909903:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318898.909911:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318898.909946:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318898.909965:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985031552 00000100:00100000:3.0:1767318898.909972:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985031552 00000100:00100000:0.0:1767318898.909999:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985031936 00000100:00100000:3.0:1767318898.910001:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985032192 00000100:00100000:3.0:1767318898.910008:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985032192 00000100:00100000:0.0:1767318898.910011:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985031936 00000100:00100000:3.0:1767318898.910024:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4664 00000100:00100000:1.0:1767318898.910238:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985032448:12345-0@lo:400:kworker.0 Request processed in 485us (633us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318898.910245:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2840 00000100:00100000:3.0:1767318898.910595:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985031552:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318898.910622:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318898.910672:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985031552:12345-0@lo:38:kworker.0 Request processed in 78us (937us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318898.911270:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4665 00000100:00100000:0.0:1767318898.911281:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985032192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318898.912012:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318898.912097:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985032192:12345-0@lo:38:kworker.0 Request processed in 814us (2150us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318898.912211:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4665 00000100:00100000:0.0:1767318898.912242:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4666 00000100:00100000:0.0:1767318898.912248:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985031936:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318898.912261:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318898.912288:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985031936:12345-0@lo:38:kworker.0 Request processed in 39us (2412us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318898.912550:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4666 00000100:00100000:0.0:1767318898.912584:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985031680 00000100:00100000:0.0:1767318898.912594:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985031680 00000100:00100000:0.0:1767318898.912628:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1656 00000100:00100000:0.0:1767318898.912633:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985031680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318898.912645:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318898.912676:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985031680:12345-0@lo:38:kworker.0 Request processed in 42us (2855us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318898.912682:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1656 00000100:00080000:0.0:1767318898.912740:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318898.912745:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318898.912748:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985031936:0@lo:38:kworker.0 00000100:00080000:0.0:1767318898.912759:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318898.912762:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318898.912765:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985032192:0@lo:38:kworker.0 00000100:00080000:0.0:1767318898.912776:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318898.912780:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318898.912783:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985031552:0@lo:38:kworker.0 00000100:00080000:0.0:1767318898.912791:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318898.912793:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318898.912796:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985031680:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.912872:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985032448:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.912908:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985032064:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.912919:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985031808:0@lo:400:kworker.0 00000100:00100000:3.0:1767318898.918755:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4664 00010000:00010000:0.0:1767318899.547364:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318900.443751:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985032704 00000100:00100000:0.0:1767318900.443766:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985032704 00000100:00100000:0.0:1767318900.443845:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985032576 00000100:00100000:0.0:1767318900.443850:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985032576 00000100:00100000:0.0:1767318900.443878:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1994 00000100:00100000:0.0:1767318900.443886:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985032704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318900.443926:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318900.444002:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985032704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (380us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318900.444010:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1994 00000100:00100000:0.0:1767318900.444026:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1995 00000100:00100000:0.0:1767318900.444032:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985032576:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318900.444046:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318900.444071:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985032576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 39us (358us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318900.444077:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1995 00000100:00080000:1.0:1767318900.444283:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318900.444288:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318900.571456:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318901.596374:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318902.619406:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318903.647760:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318903.978499:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318903.978540:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318903.978654:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920530560 00000100:00100000:1.0:1767318903.978662:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920530560 00000100:00100000:1.0:1767318903.978712:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4298 00000100:00100000:1.0:1767318903.978723:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920530688:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318903.978815:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920530688:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 95us (322us total) trans 0 rc 0/0 00000100:00080000:0.0:1767318903.979037:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318903.982461:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4298 00000100:00080000:3.0:1767318904.027441:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318904.027452:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318904.027455:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318904.027458:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318904.027462:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318904.027471:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4147 00000100:00080000:3.0:1767318904.027476:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318904.027483:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318904.027526:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7100 x1853163985032832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318904.027551:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318904.027554:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318904.027556:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318904.027559:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4147 00000100:00080000:3.0:1767318904.027561:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318904.027564:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318904.027577:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6a00 x1853163985032960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318904.027690:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985033088:0@lo:400:kworker.0 00000100:00100000:0.0:1767318904.027765:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318904.027906:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985032832:0@lo:38:kworker.0 00000100:00100000:1.0:1767318904.027938:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318904.027965:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985032960:0@lo:38:kworker.0 00000100:00100000:1.0:1767318904.027993:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318904.028038:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985032832 00000100:00100000:1.0:1767318904.028044:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985032832 00000100:00100000:1.0:1767318904.028078:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4299 00000100:00100000:1.0:1767318904.028091:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985033088:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767318904.028162:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318904.028191:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318904.028194:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318904.028206:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4147 00000100:00080000:3.0:1767318904.028209:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318904.028213:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318904.028228:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4000 x1853163985033216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318904.028256:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318904.028260:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318904.028261:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318904.028265:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4147 00000100:00080000:3.0:1767318904.028267:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318904.028270:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318904.028281:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4a80 x1853163985033472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318904.038506:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985033088:12345-0@lo:400:kworker.0 Request processed in 10408us (10729us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318904.038522:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4299 00000100:00100000:0.0:1767318904.044782:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985033088:0@lo:400:kworker.0 00000100:00100000:1.0:1767318905.563484:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318905.563505:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985033984:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318905.563562:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318905.563675:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985033984 00000100:00100000:0.0:1767318905.563684:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985033984 00000100:00100000:0.0:1767318905.563745:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1996 00000100:00100000:0.0:1767318905.563755:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985033984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318905.563786:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318905.563835:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985033984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (271us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318905.563843:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1996 00000100:00100000:1.0:1767318905.563937:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985033984:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318905.564189:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985033856 00000100:00100000:1.0:1767318905.564193:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985033856 00000100:00100000:1.0:1767318905.564244:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1997 00000100:00100000:1.0:1767318905.564252:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985033856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318905.564276:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318905.564317:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985033856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (230us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318905.564324:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1997 00010000:00010000:0.0:1767318905.695274:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318906.715903:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318907.739400:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318908.763681:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318909.097640:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318909.097738:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920530944 00000100:00100000:1.0:1767318909.097748:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920530944 00000100:00100000:1.0:1767318909.097836:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4301 00000100:00100000:1.0:1767318909.097848:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920530944:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318909.097923:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920530944:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (292us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318909.097934:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4301 00000100:00100000:0.0:1767318909.099447:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920530816 00000100:00100000:0.0:1767318909.099455:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920530816 00000100:00100000:0.0:1767318909.099493:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4671 00000100:00100000:0.0:1767318909.099499:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920530816:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767318909.099518:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318909.099577:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920530816:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 76us (220us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318909.099584:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4671 00000100:00080000:0.0:1767318909.147316:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318909.147326:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318909.147330:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318909.147333:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318909.147338:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318909.147346:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4152 00000100:00080000:0.0:1767318909.147350:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318909.147357:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318909.147422:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0700 x1853163985034112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318909.147705:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985034112 00000100:00100000:1.0:1767318909.147714:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985034112 00000100:00100000:1.0:1767318909.147751:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4672 00000100:00100000:1.0:1767318909.147759:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985034112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318909.147780:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767318909.147890:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318909.147894:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318909.147896:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318909.147900:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4152 00000100:00080000:0.0:1767318909.147903:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318909.147907:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318909.147923:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1180 x1853163985034240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318909.148598:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985034240 00000100:00100000:3.0:1767318909.148603:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985034240 00000100:00100000:3.0:1767318909.148630:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1658 00000100:00100000:3.0:1767318909.148635:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985034240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318909.148647:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318909.148678:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985034240:12345-0@lo:38:kworker.0 Request processed in 42us (273us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318909.148684:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1658 00000100:00100000:1.0:1767318909.148805:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985034112:12345-0@lo:38:kworker.0 Request processed in 1045us (1224us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318909.148812:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4672 00000100:00080000:0.0:1767318909.149252:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318909.149257:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318909.149259:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318909.149264:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4152 00000100:00080000:0.0:1767318909.149267:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318909.149272:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318909.149287:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0000 x1853163985034496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318909.149329:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985034368 00000100:00100000:1.0:1767318909.149334:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985034368 00000100:00100000:1.0:1767318909.149405:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4302 00000100:00100000:1.0:1767318909.149411:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985034368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318909.149449:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985034368:12345-0@lo:400:kworker.0 Request processed in 38us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318909.149455:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4302 00000100:00100000:1.0:1767318909.149548:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985034496 00000100:00100000:1.0:1767318909.149551:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985034496 00000100:00100000:1.0:1767318909.149566:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4673 00000100:00100000:1.0:1767318909.149571:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985034496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318909.149582:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767318909.149634:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318909.149637:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318909.149639:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318909.149643:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4152 00000100:00080000:0.0:1767318909.149645:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318909.149648:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318909.149659:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3100 x1853163985034752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318909.149717:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318909.149827:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985034624:0@lo:400:kworker.0 00000100:00100000:0.0:1767318909.149854:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318909.149888:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318909.149894:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985034880:0@lo:400:kworker.0 00000100:00100000:0.0:1767318909.149916:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318909.150076:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985034496:12345-0@lo:38:kworker.0 Request processed in 503us (613us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318909.150083:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4673 00000100:00100000:1.0:1767318909.150349:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985034624 00000100:00100000:1.0:1767318909.150353:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985034624 00000100:00100000:1.0:1767318909.150377:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985034752 00000100:00100000:1.0:1767318909.150379:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985034752 00000100:00100000:1.0:1767318909.150393:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4303 00000100:00100000:1.0:1767318909.150399:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985034624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318909.150428:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985034624:12345-0@lo:400:kworker.0 Request processed in 29us (573us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318909.150434:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4303 00000100:00100000:1.0:1767318909.150443:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4674 00000100:00100000:1.0:1767318909.150447:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985034752:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318909.150539:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318909.150609:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985035008:0@lo:400:kworker.0 00000100:00080000:3.0:1767318909.150633:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318909.150663:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985034368:0@lo:400:kworker.0 00000100:00100000:1.0:1767318909.150736:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985034752:12345-0@lo:38:kworker.0 Request processed in 287us (587us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318909.150743:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4674 00000100:00080000:3.0:1767318909.150903:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318909.150908:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318909.150911:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985034752:0@lo:38:kworker.0 00000100:00080000:3.0:1767318909.152322:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318909.152398:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318909.152482:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985035008 00000100:00100000:3.0:1767318909.152487:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985035008 00000100:00100000:3.0:1767318909.152536:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2844 00000100:00100000:3.0:1767318909.152543:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985035008:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318909.152589:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985035008:12345-0@lo:400:kworker.0 Request processed in 46us (191us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318909.152597:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2844 00000100:00080000:1.0:1767318909.152725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318909.152754:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985035008:0@lo:400:kworker.0 00000100:00100000:0.0:1767318909.152821:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985034624:0@lo:400:kworker.0 00000100:00100000:0.0:1767318909.153279:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985034880:0@lo:400:kworker.0 00010000:00010000:0.0:1767318909.788304:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318910.683449:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985035136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318910.683533:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318910.685265:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318910.685285:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985035264:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318910.685345:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318910.685435:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985035136 00000100:00100000:3.0:1767318910.685443:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985035136 00000100:00100000:3.0:1767318910.685501:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985035264 00000100:00100000:3.0:1767318910.685504:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985035264 00000100:00100000:3.0:1767318910.685522:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1998 00000100:00100000:3.0:1767318910.685530:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985035136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318910.685559:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318910.685606:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985035136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (2073us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318910.685613:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1998 00000100:00100000:3.0:1767318910.685630:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1999 00000100:00100000:3.0:1767318910.685636:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985035264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318910.685652:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318910.685680:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985035264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318910.685686:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1999 00000100:00100000:3.0:1767318910.685729:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985035136:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318910.685813:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985035264:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767318911.835483:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318912.859703:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318913.243411:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318888/real 1767318888] req@ffff8ce19db75c00 x1853163985028864/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318913 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318913.243466:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19db75c00 x1853163985028864/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318913 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318913.243481:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318913.243509:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318913.243512:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985028864:0@lo:250:kworker.0 00000100:00080000:0.0:1767318913.243636:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318913.243642:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767318913.243645:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318913.243649:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4136 00000100:00080000:0.0:1767318913.243654:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318913.243661:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318913.243693:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2680 x1853163985035392/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318913.243717:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318913.243720:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318913.243722:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318913.243726:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4157 00000100:00080000:0.0:1767318913.243728:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318913.243731:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318913.243747:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1880 x1853163985035520/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318913.243755:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318913.243758:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318913.243761:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318913.243764:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4157 00000100:00080000:0.0:1767318913.243767:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318913.243770:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318913.243780:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2d80 x1853163985035648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318913.243817:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318913.243820:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318913.243822:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318913.243826:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4157 00000100:00080000:0.0:1767318913.243828:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318913.243831:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318913.243841:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1f80 x1853163985035904/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318913.243874:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318913.243878:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318913.243880:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318913.243883:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4157 00000100:00080000:0.0:1767318913.243886:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318913.243889:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318913.243901:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3800 x1853163985036160/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318913.243938:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985036032:0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.244000:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318913.244056:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318913.244063:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985036288:0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.244088:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318913.244115:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036032 00000100:00100000:0.0:1767318913.244123:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036032 00000100:00100000:0.0:1767318913.244205:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4304 00000100:00100000:0.0:1767318913.244214:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985036032:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.244264:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985036032:12345-0@lo:400:kworker.0 Request processed in 51us (263us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318913.244271:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4304 00000100:00100000:0.0:1767318913.244308:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985036032:0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.244544:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985035776:0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.244574:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318913.244607:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318913.244613:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985036416:0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.244635:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318913.244662:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318913.244676:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985035776 00000100:00100000:1.0:1767318913.244679:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985035776 00000100:00100000:1.0:1767318913.244710:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4305 00000100:00100000:1.0:1767318913.244716:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985035776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.244749:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985035776:12345-0@lo:400:kworker.0 Request processed in 33us (175us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318913.244755:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4305 00000100:00100000:1.0:1767318913.244775:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036288 00000100:00100000:1.0:1767318913.244780:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036288 00000100:00100000:1.0:1767318913.244805:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036416 00000100:00100000:1.0:1767318913.244808:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036416 00000100:00100000:1.0:1767318913.244827:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2845 00000100:00100000:1.0:1767318913.244834:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985036288:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.244865:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985036288:12345-0@lo:400:kworker.0 Request processed in 32us (777us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318913.244871:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2845 00000100:00100000:1.0:1767318913.244882:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2846 00000100:00100000:1.0:1767318913.244887:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985036416:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.244910:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985036416:12345-0@lo:400:kworker.0 Request processed in 24us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318913.244916:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2846 00000100:00080000:1.0:1767318913.244935:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318913.244938:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318913.244940:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318913.244958:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985036416:0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.244968:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985035776:0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.245057:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985036288:0@lo:400:kworker.0 00000100:00100000:3.0:1767318913.245413:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985035392:0@lo:250:kworker.0 00000400:00000100:3.0:1767318913.245462:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985035392 offset 0 length 520: 4 00000100:00100000:3.0:1767318913.245485:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985035520:0@lo:38:kworker.0 00000100:00100000:3.0:1767318913.245509:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318913.245533:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985035648:0@lo:38:kworker.0 00000100:00100000:3.0:1767318913.245556:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318913.245624:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985035904:0@lo:38:kworker.0 00000100:00100000:3.0:1767318913.245645:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318913.245669:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985036160:0@lo:38:kworker.0 00000100:00100000:3.0:1767318913.245690:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318913.246023:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985035520 00000100:00100000:0.0:1767318913.246030:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985035520 00000100:00100000:0.0:1767318913.246062:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985035904 00000100:00100000:0.0:1767318913.246065:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985035904 00000100:00100000:0.0:1767318913.246077:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036160 00000100:00100000:0.0:1767318913.246079:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036160 00000100:00100000:0.0:1767318913.246092:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4675 00000100:00100000:0.0:1767318913.246096:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985035520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318913.246113:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318913.246145:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985035520:12345-0@lo:38:kworker.0 Request processed in 47us (635us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318913.246151:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4675 00000100:00100000:0.0:1767318913.246189:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4676 00000100:00100000:0.0:1767318913.246193:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985035904:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318913.246203:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318913.246226:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985035904:12345-0@lo:38:kworker.0 Request processed in 31us (579us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318913.246276:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4676 00000100:00100000:0.0:1767318913.246288:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4677 00000100:00100000:0.0:1767318913.246292:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985036160:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318913.246300:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318913.246320:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985036160:12345-0@lo:38:kworker.0 Request processed in 27us (630us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318913.246324:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4677 00000100:00080000:3.0:1767318913.247079:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318913.247090:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318913.247094:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985035520:0@lo:38:kworker.0 00000100:00080000:3.0:1767318913.247107:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318913.247110:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318913.247113:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985035904:0@lo:38:kworker.0 00000100:00080000:3.0:1767318913.247122:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318913.247125:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318913.247128:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985036160:0@lo:38:kworker.0 00000100:00080000:0.0:1767318913.255731:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318913.255741:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318913.255745:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985035648:0@lo:38:kworker.0 00010000:00010000:1.0:1767318913.884253:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318914.220484:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920531200 00000100:00100000:3.0:1767318914.220494:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920531200 00000100:00100000:0.0:1767318914.220628:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920531072 00000100:00100000:0.0:1767318914.220634:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920531072 00000100:00080000:1.0:1767318914.220682:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318914.220701:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4306 00000100:00100000:0.0:1767318914.220712:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920531200:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767318914.220791:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920531200:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (402us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318914.220796:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4678 00000100:00100000:0.0:1767318914.220836:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4306 00000100:00100000:3.0:1767318914.221060:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920531072:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318914.221082:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318914.221191:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920531072:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 358us (673us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318914.221200:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4678 00010000:00010000:1.0:1767318914.907494:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318915.804000:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1767318915.804023:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985036672:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318915.804128:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318915.804160:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985036544:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318915.804217:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318915.804443:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036672 00000100:00100000:3.0:1767318915.804452:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036672 00000100:00100000:3.0:1767318915.804512:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036544 00000100:00100000:3.0:1767318915.804516:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036544 00000100:00100000:3.0:1767318915.804542:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2000 00000100:00100000:3.0:1767318915.804551:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb6850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985036672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318915.804579:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318915.804621:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb6850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985036672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (492us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318915.804628:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2000 00000100:00100000:1.0:1767318915.804760:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2001 00000100:00100000:1.0:1767318915.804768:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985036544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318915.804784:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318915.804813:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985036544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (595us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318915.804818:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2001 00000100:00080000:0.0:1767318915.804992:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318915.804996:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318915.805031:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985036672:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318915.805045:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985036544:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767318915.933161:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318917.979439:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318918.363554:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318918.363563:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318918.363567:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318918.363573:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318918.363579:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318918.363590:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4161 00000100:00080000:3.0:1767318918.363596:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318918.363605:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318918.363653:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5f80 x1853163985036800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318918.363683:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318918.363686:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318918.363688:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318918.363693:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4161 00000100:00080000:3.0:1767318918.363696:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318918.363701:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318918.363712:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4e00 x1853163985036928/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318918.363761:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318918.363765:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318918.363766:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318918.363773:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4161 00000100:00080000:3.0:1767318918.363777:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318918.363783:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318918.363795:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6a00 x1853163985037184/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318918.363815:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318918.363820:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318918.363823:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318918.363829:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4161 00000100:00080000:3.0:1767318918.363831:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318918.363834:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318918.363847:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4a80 x1853163985037440/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318918.363929:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985037568:0@lo:400:kworker.0 00000100:00100000:3.0:1767318918.363978:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318918.364014:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318918.364026:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985037696:0@lo:400:kworker.0 00000100:00100000:3.0:1767318918.364057:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318918.364092:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985037056 00000100:00100000:3.0:1767318918.364097:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985037056 00000100:00100000:3.0:1767318918.364131:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985037312 00000100:00100000:3.0:1767318918.364140:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985037312 00000100:00100000:3.0:1767318918.364204:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4307 00000100:00100000:3.0:1767318918.364212:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985037056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318918.364273:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985037056:12345-0@lo:400:kworker.0 Request processed in 58us (367us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318918.364288:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4307 00000100:00100000:3.0:1767318918.364310:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4308 00000100:00100000:3.0:1767318918.364326:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985037312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318918.364371:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985037312:12345-0@lo:400:kworker.0 Request processed in 54us (370us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318918.364376:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4308 00000100:00080000:0.0:1767318918.364469:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318918.364474:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318918.364476:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318918.364492:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985036800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318918.364530:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318918.364559:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985036928:0@lo:38:kworker.0 00000100:00100000:0.0:1767318918.364585:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318918.364644:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985037696:0@lo:400:kworker.0 00000100:00100000:3.0:1767318918.364659:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985037568:0@lo:400:kworker.0 00000100:00100000:0.0:1767318918.364701:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985037184:0@lo:38:kworker.0 00000100:00100000:0.0:1767318918.364736:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318918.364754:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985037440:0@lo:38:kworker.0 00000100:00100000:0.0:1767318918.364774:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318918.364800:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036928 00000100:00100000:0.0:1767318918.364803:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036928 00000100:00100000:0.0:1767318918.364825:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1660 00000100:00100000:0.0:1767318918.364830:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985036928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318918.364846:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318918.364875:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985036928:12345-0@lo:38:kworker.0 Request processed in 44us (290us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318918.364879:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1660 00000100:00100000:0.0:1767318918.364894:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985036800 00000100:00100000:0.0:1767318918.364897:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985036800 00000100:00100000:0.0:1767318918.364907:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985037184 00000100:00100000:0.0:1767318918.364908:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985037184 00000100:00100000:0.0:1767318918.364914:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985037440 00000100:00100000:0.0:1767318918.364915:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985037440 00000100:00100000:0.0:1767318918.364923:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4679 00000100:00100000:0.0:1767318918.364925:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985036800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318918.364931:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318918.364946:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985036800:12345-0@lo:38:kworker.0 Request processed in 19us (416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318918.364949:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4679 00000100:00100000:0.0:1767318918.364954:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4680 00000100:00100000:0.0:1767318918.364957:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985037184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318918.364961:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318918.364970:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985037184:12345-0@lo:38:kworker.0 Request processed in 13us (235us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318918.364974:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4680 00000100:00100000:0.0:1767318918.364978:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4681 00000100:00100000:0.0:1767318918.364980:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985037440:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318918.364985:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318918.364994:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985037440:12345-0@lo:38:kworker.0 Request processed in 12us (221us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318918.364999:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4681 00000100:00080000:0.0:1767318918.365016:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318918.365020:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318918.365022:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985036800:0@lo:38:kworker.0 00000100:00080000:0.0:1767318918.365026:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318918.365028:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318918.365029:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985036928:0@lo:38:kworker.0 00000100:00080000:0.0:1767318918.365033:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318918.365035:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318918.365037:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985037184:0@lo:38:kworker.0 00000100:00080000:0.0:1767318918.365040:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318918.365041:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318918.365043:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985037440:0@lo:38:kworker.0 00010000:00010000:1.0:1767318919.003416:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318919.343234:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920531584 00000100:00100000:1.0:1767318919.343245:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920531584 00000100:00100000:1.0:1767318919.343313:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920531456 00000100:00100000:1.0:1767318919.343316:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920531456 00000100:00100000:1.0:1767318919.343353:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4309 00000100:00100000:1.0:1767318919.343362:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920531584:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318919.343457:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920531584:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 95us (988us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318919.343466:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4309 00000100:00100000:1.0:1767318919.343483:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4682 00000100:00100000:1.0:1767318919.343488:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920531456:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318919.343512:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318919.343540:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920531456:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 50us (459us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318919.343546:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4682 00000100:00080000:0.0:1767318919.344456:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318920.027503:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318920.923814:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985037824 00000100:00100000:1.0:1767318920.923823:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985037824 00000100:00100000:1.0:1767318920.923896:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2002 00000100:00100000:1.0:1767318920.923906:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985037824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318920.923936:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318920.923981:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985037824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (371us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318920.923989:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2002 00000100:00100000:3.0:1767318920.924319:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318920.924329:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985037952:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318920.924368:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318920.924395:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985037952 00000100:00100000:3.0:1767318920.924646:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985037952 00000100:00100000:3.0:1767318920.924682:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2003 00000100:00100000:3.0:1767318920.924687:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985037952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318920.924700:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318920.924731:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985037952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (362us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318920.924737:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2003 00000100:00100000:3.0:1767318920.924766:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985037952:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318921.052028:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318922.075501:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318923.483355:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318923.483366:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318923.483370:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318923.483375:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318923.483382:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318923.483390:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4166 00000100:00080000:1.0:1767318923.483395:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318923.483401:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318923.483443:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630000 x1853163985038080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318923.483475:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318923.483480:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318923.483482:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318923.483486:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4166 00000100:00080000:1.0:1767318923.483489:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318923.483493:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318923.483508:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631880 x1853163985038208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318923.483552:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318923.483558:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318923.483559:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318923.483563:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4166 00000100:00080000:1.0:1767318923.483566:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318923.483570:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318923.483583:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632680 x1853163985038464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318923.483604:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318923.483609:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318923.483615:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318923.483619:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4166 00000100:00080000:1.0:1767318923.483621:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318923.483626:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318923.483638:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630a80 x1853163985038720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318923.483694:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985038848:0@lo:400:kworker.0 00000100:00100000:1.0:1767318923.483737:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318923.483871:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985038592:0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.483905:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318923.483936:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318923.483947:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985038336:0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.483972:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318923.484007:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985038592 00000100:00100000:3.0:1767318923.484012:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985038592 00000100:00100000:3.0:1767318923.484054:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985038336 00000100:00100000:3.0:1767318923.484059:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985038336 00000100:00100000:3.0:1767318923.484082:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4311 00000100:00100000:3.0:1767318923.484091:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985038336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.484137:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985038336:12345-0@lo:400:kworker.0 Request processed in 46us (163us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318923.484149:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4311 00000100:00100000:0.0:1767318923.484307:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985038976:0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.484391:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4685 00000100:00100000:3.0:1767318923.484397:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985038720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318923.484409:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318923.484497:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985038848 00000100:00100000:0.0:1767318923.484501:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985038848 00000100:00100000:0.0:1767318923.484531:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2849 00000100:00100000:0.0:1767318923.484537:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985038848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.487580:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985038720:12345-0@lo:38:kworker.0 Request processed in 3172us (3747us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318923.487594:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4685 00000100:00100000:3.0:1767318923.487693:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985038208 00000100:00100000:3.0:1767318923.487701:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985038208 00000100:00100000:3.0:1767318923.487742:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1661 00000100:00100000:3.0:1767318923.487747:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985038208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318923.487770:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318923.487809:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985038208:12345-0@lo:38:kworker.0 Request processed in 60us (4106us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318923.487815:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1661 00000100:00080000:3.0:1767318923.487839:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318923.487845:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318923.487877:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318923.487882:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318923.487886:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985038208:0@lo:38:kworker.0 00000100:00080000:3.0:1767318923.487896:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318923.487899:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318923.487902:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985038720:0@lo:38:kworker.0 00000100:00080000:3.0:1767318923.487912:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318923.487916:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318923.487919:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985038080:0@lo:38:kworker.0 00000100:00100000:3.0:1767318923.487973:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985038336:0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.487985:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985038592:0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.488195:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318923.488239:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985038976 00000100:00100000:3.0:1767318923.488243:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985038976 00000100:00100000:3.0:1767318923.488504:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2850 00000100:00100000:3.0:1767318923.488512:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985038976:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318923.488565:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985038976:12345-0@lo:400:kworker.0 Request processed in 54us (369us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318923.488572:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2850 00000100:00080000:3.0:1767318923.488591:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318923.488607:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985038976:0@lo:400:kworker.0 00000100:00080000:3.0:1767318923.494662:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318923.494672:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318923.494677:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985038464:0@lo:38:kworker.0 00000100:00100000:0.0:1767318923.494974:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985038848:12345-0@lo:400:kworker.0 Request processed in 10434us (11236us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318923.494986:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2849 00000100:00100000:1.0:1767318923.495096:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985038848:0@lo:400:kworker.0 00010000:00010000:0.0:1767318924.124788:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318924.457941:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920531840 00000100:00100000:1.0:1767318924.457951:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920531840 00000100:00100000:3.0:1767318924.457981:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920531712 00000100:00100000:3.0:1767318924.457986:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920531712 00000100:00100000:3.0:1767318924.458026:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4686 00000100:00100000:3.0:1767318924.458034:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920531712:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318924.458063:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318924.458086:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4312 00000100:00100000:1.0:1767318924.458092:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920531840:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318924.458127:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920531840:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 36us (322us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318924.458127:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920531712:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 93us (254us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318924.458135:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4686 00000100:00080000:0.0:1767318924.458500:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318924.458552:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4312 00010000:00010000:1.0:1767318925.147982:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318926.043546:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985039104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318926.043622:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318926.043992:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985039104 00000100:00100000:3.0:1767318926.044002:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985039104 00000100:00100000:3.0:1767318926.044074:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2004 00000100:00100000:3.0:1767318926.044084:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985039104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318926.044117:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318926.045085:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985039104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (543us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318926.045096:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2004 00000100:00080000:0.0:1767318926.045344:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318926.045392:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985039104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318926.045410:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318926.045421:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985039232:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318926.045462:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318926.047550:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985039232:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318926.047774:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318926.173934:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318927.195400:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318928.219405:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318928.603298:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318928.603310:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318928.603314:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318928.603318:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318928.603323:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318928.603331:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4171 00000100:00080000:3.0:1767318928.603336:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318928.603343:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318928.603384:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5500 x1853163985039360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318928.603400:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318928.603404:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318928.603406:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318928.603409:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4171 00000100:00080000:3.0:1767318928.603411:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318928.603414:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318928.603428:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4a80 x1853163985039488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318928.603456:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318928.603460:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318928.603461:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318928.603465:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4171 00000100:00080000:3.0:1767318928.603467:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318928.603470:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318928.603481:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4e00 x1853163985039744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318928.603495:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318928.603498:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318928.603500:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318928.603503:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4171 00000100:00080000:3.0:1767318928.603505:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318928.603508:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318928.603521:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7800 x1853163985040000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318928.603545:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985039616:0@lo:400:kworker.0 00000100:00100000:1.0:1767318928.603611:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318928.603674:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318928.603680:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985039872:0@lo:400:kworker.0 00000100:00100000:3.0:1767318928.603691:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985040128:0@lo:400:kworker.0 00000100:00100000:1.0:1767318928.603701:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318928.603721:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318928.603730:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985039616 00000100:00100000:1.0:1767318928.603737:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985039616 00000100:00100000:3.0:1767318928.603741:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318928.603745:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985040256:0@lo:400:kworker.0 00000100:00100000:3.0:1767318928.603764:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318928.603790:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985040128 00000100:00100000:1.0:1767318928.603793:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985039872 00000100:00100000:3.0:1767318928.603793:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985040128 00000100:00100000:1.0:1767318928.603795:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985039872 00000100:00100000:1.0:1767318928.603817:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4313 00000100:00100000:3.0:1767318928.603817:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985040256 00000100:00100000:3.0:1767318928.603820:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985040256 00000100:00100000:1.0:1767318928.603825:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985039616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318928.603838:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2851 00000100:00100000:3.0:1767318928.603843:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985040128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318928.603873:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985039616:12345-0@lo:400:kworker.0 Request processed in 47us (261us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318928.603873:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985040128:12345-0@lo:400:kworker.0 Request processed in 29us (150us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318928.603881:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4313 00000100:00100000:3.0:1767318928.603881:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2851 00000100:00100000:3.0:1767318928.603896:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2852 00000100:00100000:3.0:1767318928.603902:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985040256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318928.603910:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4314 00000100:00100000:1.0:1767318928.603916:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985039872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318928.603927:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985040256:12345-0@lo:400:kworker.0 Request processed in 26us (162us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318928.603932:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2852 00000100:00100000:3.0:1767318928.603973:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985040256:0@lo:400:kworker.0 00000100:00100000:3.0:1767318928.603982:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985040128:0@lo:400:kworker.0 00000100:00080000:3.0:1767318928.604089:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318928.604093:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318928.604095:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318928.604265:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985039872:12345-0@lo:400:kworker.0 Request processed in 349us (562us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318928.604272:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4314 00000100:00100000:1.0:1767318928.604300:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985039872:0@lo:400:kworker.0 00000100:00100000:1.0:1767318928.604308:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985039616:0@lo:400:kworker.0 00000100:00100000:0.0:1767318928.604686:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985039360 00000100:00100000:0.0:1767318928.604692:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985039360 00000100:00100000:0.0:1767318928.604723:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985039744 00000100:00100000:0.0:1767318928.604726:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985039744 00000100:00100000:0.0:1767318928.604737:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985040000 00000100:00100000:0.0:1767318928.604739:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985040000 00000100:00100000:0.0:1767318928.604756:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4687 00000100:00100000:0.0:1767318928.604761:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985039360:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318928.604788:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318928.604825:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985039360:12345-0@lo:38:kworker.0 Request processed in 63us (319us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318928.604831:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4687 00000100:00100000:0.0:1767318928.604843:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4688 00000100:00100000:0.0:1767318928.604846:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985039744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318928.604854:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318928.604876:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985039744:12345-0@lo:38:kworker.0 Request processed in 28us (271us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318928.604882:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4688 00000100:00100000:3.0:1767318928.604888:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985039488 00000100:00100000:0.0:1767318928.604891:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4689 00000100:00100000:3.0:1767318928.604892:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985039488 00000100:00100000:0.0:1767318928.604895:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985040000:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318928.604914:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1662 00010000:00020000:0.0:1767318928.604918:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318928.604918:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985039488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318928.604927:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318928.604956:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985039488:12345-0@lo:38:kworker.0 Request processed in 36us (397us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318928.604961:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1662 00000100:00080000:3.0:1767318928.605014:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318928.605019:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318928.605023:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985039360:0@lo:38:kworker.0 00000100:00080000:3.0:1767318928.605031:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318928.605035:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318928.605037:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985039488:0@lo:38:kworker.0 00000100:00080000:3.0:1767318928.605044:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318928.605047:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318928.605049:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985039744:0@lo:38:kworker.0 00000100:00100000:0.0:1767318928.605250:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985040000:12345-0@lo:38:kworker.0 Request processed in 353us (598us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318928.605256:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4689 00000100:00080000:3.0:1767318928.607784:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318928.607796:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318928.607801:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985040000:0@lo:38:kworker.0 00000100:00100000:1.0:1767318929.583240:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318929.583291:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:3.0:1767318929.583875:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767318930.267397:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318931.163546:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318931.163571:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985040384:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318931.163640:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318931.163965:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985040384 00000100:00100000:3.0:1767318931.163973:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985040384 00000100:00100000:3.0:1767318931.164044:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985040512 00000100:00100000:3.0:1767318931.164048:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985040512 00000100:00100000:3.0:1767318931.164074:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2006 00000100:00100000:3.0:1767318931.164084:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985040384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318931.164125:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318931.164197:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985040384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 115us (556us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318931.164205:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2006 00000100:00100000:3.0:1767318931.164223:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2007 00000100:00100000:3.0:1767318931.164229:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985040512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318931.164244:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318931.164274:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985040512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 45us (399us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318931.164280:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2007 00000100:00100000:3.0:1767318931.164328:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985040384:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318931.164366:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985040512:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318931.291807:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318932.317190:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318933.346384:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318933.725106:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318933.725117:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318933.725120:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318933.725124:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318933.725128:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318933.725136:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4176 00000100:00080000:0.0:1767318933.725140:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318933.725147:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318933.725204:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3800 x1853163985040640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318933.725232:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318933.725236:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318933.725238:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318933.725241:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4176 00000100:00080000:0.0:1767318933.725244:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318933.725248:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318933.725261:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1f80 x1853163985040768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318933.725302:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318933.725305:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318933.725308:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318933.725312:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4176 00000100:00080000:0.0:1767318933.725314:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318933.725318:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318933.725330:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985041024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318933.725347:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318933.725350:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318933.725352:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318933.725355:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4176 00000100:00080000:0.0:1767318933.725357:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318933.725361:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318933.725374:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5180 x1853163985041280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318933.725720:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985040640:0@lo:38:kworker.0 00000100:00100000:3.0:1767318933.725749:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318933.725831:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985040768:0@lo:38:kworker.0 00000100:00100000:3.0:1767318933.725857:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318933.725878:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985041024:0@lo:38:kworker.0 00000100:00100000:3.0:1767318933.725897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318933.729003:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985041280:0@lo:38:kworker.0 00000100:00100000:3.0:1767318933.730215:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318933.733110:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318933.733113:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318933.734967:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318933.734979:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318933.734983:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985040640:0@lo:38:kworker.0 00000100:00080000:3.0:1767318933.734996:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318933.735000:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318933.735003:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985040768:0@lo:38:kworker.0 00000100:00080000:3.0:1767318933.735013:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318933.735016:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318933.735019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985041024:0@lo:38:kworker.0 00000100:00100000:1.0:1767318933.735109:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985041280 00000100:00100000:1.0:1767318933.735117:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985041280 00000100:00100000:1.0:1767318933.735159:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4693 00000100:00100000:1.0:1767318933.735182:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985041280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318933.735200:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318933.735257:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985041280:12345-0@lo:38:kworker.0 Request processed in 75us (5048us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318933.735264:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4693 00000100:00080000:3.0:1767318933.735323:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318933.735327:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318933.735329:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985041280:0@lo:38:kworker.0 00010000:00010000:0.0:1767318934.364391:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318934.704205:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920532224 00000100:00100000:1.0:1767318934.704215:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920532224 00000100:00100000:1.0:1767318934.704259:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920532352 00000100:00100000:1.0:1767318934.704262:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920532352 00000100:00100000:1.0:1767318934.704311:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4318 00000100:00100000:1.0:1767318934.704320:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920532352:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318934.704405:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920532352:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (596us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318934.704412:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4318 00000100:00080000:0.0:1767318934.704502:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318934.704530:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4694 00000100:00100000:0.0:1767318934.704535:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920532224:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767318934.704558:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318934.704599:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920532224:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 62us (1203us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318934.704607:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4694 00000100:00100000:0.0:1767318936.283489:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985041664:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318936.283567:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318936.283749:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318936.283758:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985041792:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318936.283783:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318936.283859:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985041664 00000100:00100000:1.0:1767318936.283866:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985041664 00000100:00100000:1.0:1767318936.283913:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985041792 00000100:00100000:1.0:1767318936.283916:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985041792 00000100:00100000:1.0:1767318936.283928:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2008 00000100:00100000:1.0:1767318936.283934:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985041664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318936.283961:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318936.284007:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985041664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (440us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318936.284030:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2008 00000100:00100000:1.0:1767318936.284047:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2009 00000100:00100000:1.0:1767318936.284054:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985041792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318936.284068:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318936.284088:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985041792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 36us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318936.284092:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2009 00000100:00100000:1.0:1767318936.284132:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985041792:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318936.284141:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985041664:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318936.284293:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318936.284297:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767318936.413382:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318937.436191:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318938.459482:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318938.795533:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920532736 00000100:00100000:3.0:1767318938.795544:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920532736 00000100:00100000:3.0:1767318938.795610:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920532608 00000100:00100000:3.0:1767318938.795613:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920532608 00000100:00100000:3.0:1767318938.795637:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4319 00000100:00100000:3.0:1767318938.795645:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920532736:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318938.795713:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920532736:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 69us (373us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318938.795720:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4319 00000100:00100000:3.0:1767318938.795737:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4695 00000100:00100000:3.0:1767318938.795741:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920532608:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318938.795759:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318938.795779:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920532608:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 37us (375us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318938.795785:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4695 00000100:00080000:3.0:1767318938.844187:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318938.844193:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767318938.844195:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318938.844200:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4161 00000100:00080000:3.0:1767318938.844204:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318938.844211:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318938.844244:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6a00 x1853163985041920/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318938.844277:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318938.844280:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318938.844282:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318938.844286:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4181 00000100:00080000:3.0:1767318938.844288:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318938.844290:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318938.844313:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5f80 x1853163985042048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318938.844321:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318938.844324:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318938.844325:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318938.844328:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4181 00000100:00080000:3.0:1767318938.844330:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318938.844333:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318938.844345:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5c00 x1853163985042176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318938.844480:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985042304:0@lo:400:kworker.0 00000100:00100000:0.0:1767318938.844545:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318938.844651:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985042304 00000100:00100000:1.0:1767318938.844667:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985042304 00000100:00100000:1.0:1767318938.844718:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4320 00000100:00100000:1.0:1767318938.844725:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985042304:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767318938.845467:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318938.845472:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318938.845475:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318938.845479:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4181 00000100:00080000:3.0:1767318938.845481:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318938.845486:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318938.845502:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5f80 x1853163985042432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318938.860902:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985042304:12345-0@lo:400:kworker.0 Request processed in 16165us (16345us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318938.860917:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4320 00000100:00100000:0.0:1767318938.861692:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985042304:0@lo:400:kworker.0 00000100:00080000:3.0:1767318938.861872:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318938.861879:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318938.861883:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318938.861888:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4181 00000100:00080000:3.0:1767318938.861891:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318938.861897:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318938.861948:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5500 x1853163985042688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318938.862050:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985042944:0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.862108:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318938.862138:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318938.862145:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985042816:0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.862186:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318938.862214:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985042944 00000100:00100000:3.0:1767318938.862220:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985042944 00000100:00100000:3.0:1767318938.862282:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985042816 00000100:00100000:3.0:1767318938.862286:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985042816 00000100:00100000:3.0:1767318938.862896:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2855 00000100:00100000:3.0:1767318938.862906:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985042944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.862976:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985042944:12345-0@lo:400:kworker.0 Request processed in 70us (866us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318938.862983:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2855 00000100:00100000:3.0:1767318938.863000:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2856 00000100:00100000:3.0:1767318938.863006:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985042816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.863035:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985042816:12345-0@lo:400:kworker.0 Request processed in 29us (849us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318938.863041:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2856 00000100:00100000:3.0:1767318938.863073:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985042816:0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.863083:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985042944:0@lo:400:kworker.0 00000100:00100000:1.0:1767318938.863282:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985042688 00000100:00100000:1.0:1767318938.863288:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985042688 00000100:00100000:1.0:1767318938.863333:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4698 00000100:00100000:1.0:1767318938.863338:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985042688:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318938.863356:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318938.864100:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318938.864106:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318938.864109:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985042688:0@lo:38:kworker.0 00010000:00010000:1.0:1767318939.490888:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318940.508430:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318941.403554:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318941.403580:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985043072:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318941.403648:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318941.403710:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043072 00000100:00100000:1.0:1767318941.403717:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043072 00000100:00100000:1.0:1767318941.404138:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2010 00000100:00100000:1.0:1767318941.404148:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985043072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318941.404196:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318941.404243:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985043072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (594us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318941.404250:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2010 00000100:00100000:1.0:1767318941.404294:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985043072:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318941.408193:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043200 00000100:00100000:3.0:1767318941.408203:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043200 00000100:00100000:3.0:1767318941.408267:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2011 00000100:00100000:3.0:1767318941.408275:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985043200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318941.408306:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318941.408364:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985043200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (1076us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318941.408371:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2011 00000100:00100000:0.0:1767318941.408511:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985043200:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767318941.531732:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318942.558478:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318943.579926:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318943.915802:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318943.915849:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318943.915914:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920532992 00000100:00100000:3.0:1767318943.915924:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920532992 00000100:00100000:3.0:1767318943.916004:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920532864 00000100:00100000:3.0:1767318943.916008:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920532864 00000100:00100000:2.0:1767318943.916015:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4322 00000100:00100000:2.0:1767318943.916032:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920532992:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767318943.916066:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318943.916118:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920532992:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 94us (328us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318943.924439:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4699 00000100:00100000:3.0:1767318943.924454:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920532864:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318943.924480:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318943.924658:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920532864:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 202us (8805us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318943.924667:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4699 00000100:00100000:2.0:1767318943.924963:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4322 00000100:00080000:2.0:1767318943.963709:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.963727:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318943.963730:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.963734:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.963739:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.963747:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.963751:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.963758:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318943.963798:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adce00 x1853163985043328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318943.963858:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985043328:0@lo:38:kworker.0 00000100:00100000:3.0:1767318943.963943:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318943.964066:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.964070:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.964072:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.964075:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.964078:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.964081:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318943.964097:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adf480 x1853163985043456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318943.964185:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043328 00000100:00100000:1.0:1767318943.964195:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043328 00000100:00100000:1.0:1767318943.964243:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4700 00000100:00100000:1.0:1767318943.964251:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985043328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318943.964273:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318943.964497:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985043456:0@lo:38:kworker.0 00000100:00100000:3.0:1767318943.964526:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.965528:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043456 00000100:00100000:0.0:1767318943.965540:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043456 00000100:00100000:0.0:1767318943.965578:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1665 00000100:00100000:0.0:1767318943.965585:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985043456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318943.965603:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767318943.965611:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.965615:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.965617:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.965621:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.965625:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.965628:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318943.965645:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88e00 x1853163985043712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318943.965653:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985043456:12345-0@lo:38:kworker.0 Request processed in 68us (1127us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318943.965660:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1665 00000100:00080000:2.0:1767318943.965661:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.965664:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.965665:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.965668:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.965670:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.965673:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318943.965683:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ed80 x1853163985043968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318943.965711:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6aded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985043584:0@lo:400:kworker.0 00000100:00100000:2.0:1767318943.965742:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985044096:0@lo:400:kworker.0 00000100:00100000:0.0:1767318943.965743:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.965767:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318943.965770:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.965772:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985043840:0@lo:400:kworker.0 00000100:00100000:2.0:1767318943.965788:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318943.965790:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318943.965793:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985044224:0@lo:400:kworker.0 00000100:00100000:2.0:1767318943.965812:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.965816:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043584 00000100:00100000:0.0:1767318943.965819:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043584 00000100:00100000:2.0:1767318943.965835:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044096 00000100:00100000:2.0:1767318943.965839:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044096 00000100:00100000:0.0:1767318943.965855:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043840 00000100:00100000:0.0:1767318943.965857:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043840 00000100:00100000:2.0:1767318943.965863:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044224 00000100:00100000:2.0:1767318943.965865:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044224 00000100:00100000:0.0:1767318943.965890:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4323 00000100:00100000:0.0:1767318943.965897:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985043584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318943.965898:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2857 00000100:00100000:2.0:1767318943.965904:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985044096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318943.965935:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985043584:12345-0@lo:400:kworker.0 Request processed in 38us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318943.965935:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985044096:12345-0@lo:400:kworker.0 Request processed in 31us (163us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318943.965941:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4323 00000100:00100000:2.0:1767318943.965941:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2857 00000100:00100000:0.0:1767318943.965954:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4324 00000100:00100000:2.0:1767318943.965954:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2858 00000100:00100000:0.0:1767318943.965960:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985043840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318943.965960:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985044224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318943.965983:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985043840:12345-0@lo:400:kworker.0 Request processed in 23us (191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318943.965983:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985044224:12345-0@lo:400:kworker.0 Request processed in 24us (171us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318943.965989:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4324 00000100:00100000:2.0:1767318943.965989:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2858 00000100:00080000:0.0:1767318943.966008:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318943.966012:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318943.966014:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318943.966038:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985043840:0@lo:400:kworker.0 00000100:00100000:2.0:1767318943.966038:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985044224:0@lo:400:kworker.0 00000100:00100000:0.0:1767318943.966047:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6aded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985043584:0@lo:400:kworker.0 00000100:00100000:2.0:1767318943.966047:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985044096:0@lo:400:kworker.0 00000100:00100000:1.0:1767318943.967771:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985043328:12345-0@lo:38:kworker.0 Request processed in 3511us (3819us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318943.967786:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4700 00000100:00080000:3.0:1767318943.974186:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318943.974199:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318943.974205:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985043328:0@lo:38:kworker.0 00000100:00100000:3.0:1767318943.974244:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985043712:0@lo:38:kworker.0 00000100:00100000:3.0:1767318943.974311:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318943.974355:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985043968:0@lo:38:kworker.0 00000100:00100000:3.0:1767318943.974377:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318943.974416:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318943.974420:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318943.974424:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985043456:0@lo:38:kworker.0 00000100:00100000:0.0:1767318943.974529:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043712 00000100:00100000:0.0:1767318943.974537:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043712 00000100:00100000:0.0:1767318943.974575:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985043968 00000100:00100000:0.0:1767318943.974578:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985043968 00000100:00100000:0.0:1767318943.974607:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4701 00000100:00100000:0.0:1767318943.974614:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985043712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318943.974636:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318943.974681:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985043712:12345-0@lo:38:kworker.0 Request processed in 66us (369us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318943.974688:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4701 00000100:00100000:0.0:1767318943.974705:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4702 00000100:00100000:0.0:1767318943.974708:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985043968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318943.974730:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318943.974752:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985043968:12345-0@lo:38:kworker.0 Request processed in 41us (374us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318943.974758:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4702 00000100:00080000:3.0:1767318943.975037:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318943.975042:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318943.975045:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985043712:0@lo:38:kworker.0 00000100:00080000:3.0:1767318943.975053:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318943.975057:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318943.975059:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985043968:0@lo:38:kworker.0 00010000:00010000:1.0:1767318944.603410:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318945.627401:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318946.526352:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985044352:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318946.526424:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318946.526524:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044352 00000100:00100000:1.0:1767318946.526532:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044352 00000100:00100000:1.0:1767318946.526599:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2012 00000100:00100000:1.0:1767318946.526609:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985044352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318946.526637:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318946.526689:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985044352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318946.526696:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2012 00000100:00100000:2.0:1767318946.526723:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985044352:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318946.526753:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318946.526981:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985044480:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318946.527010:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318946.527083:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044480 00000100:00100000:1.0:1767318946.527087:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044480 00000100:00100000:1.0:1767318946.527139:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2013 00000100:00100000:1.0:1767318946.527145:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985044480:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318946.527161:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318946.527231:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985044480:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (220us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318946.527237:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2013 00000100:00080000:0.0:1767318946.527527:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318946.527971:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985044480:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767318946.652088:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318947.675353:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318948.705584:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318949.046029:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767318949.046077:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318949.046317:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920533248 00000100:00100000:3.0:1767318949.046328:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920533248 00000100:00100000:3.0:1767318949.046407:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920533120 00000100:00100000:3.0:1767318949.046411:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920533120 00000100:00100000:3.0:1767318949.046428:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4325 00000100:00100000:3.0:1767318949.046438:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920533248:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318949.046510:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920533248:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 73us (491us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318949.046519:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4325 00000100:00100000:3.0:1767318949.046536:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4703 00000100:00100000:3.0:1767318949.046541:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920533120:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318949.046560:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318949.046585:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920533120:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (507us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318949.046591:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4703 00000100:00080000:1.0:1767318949.047074:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318949.083874:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318949.083885:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318949.083888:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318949.083891:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318949.083895:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318949.083903:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4192 00000100:00080000:2.0:1767318949.083907:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318949.083914:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318949.083965:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75f80 x1853163985044608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318949.083994:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318949.083998:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318949.084000:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318949.084003:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4192 00000100:00080000:2.0:1767318949.084006:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318949.084009:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318949.084021:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985044736/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318949.084071:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318949.084075:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318949.084076:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318949.084080:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4192 00000100:00080000:2.0:1767318949.084082:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318949.084086:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318949.084096:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985044992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318949.084113:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318949.084116:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318949.084118:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318949.084121:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4192 00000100:00080000:2.0:1767318949.084124:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318949.084127:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318949.084195:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985045248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318949.084316:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985044864:0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.084395:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318949.084456:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318949.084463:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985045120:0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.084484:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318949.084511:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985045376:0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.084538:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318949.084561:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318949.084566:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985045504:0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.084586:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318949.084619:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985045376 00000100:00100000:0.0:1767318949.084625:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985045376 00000100:00100000:0.0:1767318949.084677:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985045504 00000100:00100000:0.0:1767318949.084680:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985045504 00000100:00100000:0.0:1767318949.084705:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2859 00000100:00100000:0.0:1767318949.084713:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985045376:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.084766:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985045376:12345-0@lo:400:kworker.0 Request processed in 54us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318949.084785:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2859 00000100:00100000:0.0:1767318949.084802:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2860 00000100:00100000:0.0:1767318949.084808:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985045504:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.084835:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985045504:12345-0@lo:400:kworker.0 Request processed in 27us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318949.084841:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2860 00000100:00100000:0.0:1767318949.084869:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044864 00000100:00100000:0.0:1767318949.084873:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044864 00000100:00100000:0.0:1767318949.084896:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985045120 00000100:00100000:0.0:1767318949.084899:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985045120 00000100:00100000:0.0:1767318949.084916:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4326 00000100:00100000:0.0:1767318949.084922:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985044864:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.084956:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985044864:12345-0@lo:400:kworker.0 Request processed in 34us (561us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318949.084961:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4326 00000100:00100000:0.0:1767318949.084974:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4327 00000100:00100000:0.0:1767318949.084979:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985045120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.085005:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985045120:12345-0@lo:400:kworker.0 Request processed in 26us (521us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318949.085011:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4327 00000100:00100000:0.0:1767318949.085046:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985045504:0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.085056:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985045376:0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.085081:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985045120:0@lo:400:kworker.0 00000100:00100000:0.0:1767318949.085089:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985044864:0@lo:400:kworker.0 00000100:00100000:3.0:1767318949.085242:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985044608:0@lo:38:kworker.0 00000100:00100000:3.0:1767318949.085274:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318949.085305:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985044736:0@lo:38:kworker.0 00000100:00100000:3.0:1767318949.085327:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318949.085359:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985044992:0@lo:38:kworker.0 00000100:00100000:3.0:1767318949.085405:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318949.085423:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985045248:0@lo:38:kworker.0 00000100:00100000:3.0:1767318949.085444:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318949.085473:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044736 00000100:00100000:3.0:1767318949.085477:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044736 00000100:00100000:3.0:1767318949.085499:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1666 00000100:00100000:3.0:1767318949.085504:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985044736:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318949.085521:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318949.085556:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985044736:12345-0@lo:38:kworker.0 Request processed in 50us (227us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318949.085562:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1666 00000100:00100000:3.0:1767318949.085580:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044608 00000100:00100000:3.0:1767318949.085583:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044608 00000100:00100000:3.0:1767318949.085595:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985044992 00000100:00100000:3.0:1767318949.085597:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985044992 00000100:00100000:3.0:1767318949.085605:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985045248 00000100:00100000:3.0:1767318949.085607:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985045248 00000100:00100000:3.0:1767318949.085616:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4704 00000100:00100000:3.0:1767318949.085620:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce18b157850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985044608:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318949.085630:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318949.085653:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce18b157850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985044608:12345-0@lo:38:kworker.0 Request processed in 31us (379us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318949.085659:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4704 00000100:00100000:3.0:1767318949.086708:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4705 00000100:00100000:3.0:1767318949.086715:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985044992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318949.086730:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318949.086784:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985044992:12345-0@lo:38:kworker.0 Request processed in 56us (1366us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318949.086792:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4705 00000100:00100000:3.0:1767318949.086805:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4706 00000100:00100000:3.0:1767318949.086809:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985045248:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318949.086819:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318949.086841:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985045248:12345-0@lo:38:kworker.0 Request processed in 31us (1397us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318949.086848:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4706 00000100:00080000:3.0:1767318949.086897:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318949.086902:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318949.086906:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985044608:0@lo:38:kworker.0 00000100:00080000:3.0:1767318949.086917:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318949.086920:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318949.086924:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985044736:0@lo:38:kworker.0 00000100:00080000:3.0:1767318949.086932:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318949.086936:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318949.086938:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985044992:0@lo:38:kworker.0 00000100:00080000:3.0:1767318949.086946:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318949.086949:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318949.086952:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985045248:0@lo:38:kworker.0 00000100:00080000:1.0:1767318949.087312:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318949.087319:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318949.087322:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318949.725625:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318950.749055:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318951.643413:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985045632:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318951.643488:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318951.643638:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985045632 00000100:00100000:2.0:1767318951.643647:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985045632 00000100:00100000:2.0:1767318951.643707:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2014 00000100:00100000:2.0:1767318951.643718:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985045632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318951.643769:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767318951.643861:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318951.647750:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985045632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4022us (4249us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318951.648317:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985045760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318951.648388:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318951.648438:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985045760 00000100:00100000:0.0:1767318951.648447:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985045760 00000100:00100000:0.0:1767318951.648532:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2015 00000100:00100000:0.0:1767318951.648545:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985045760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318951.648578:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318951.648619:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985045760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318951.648626:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2015 00000100:00100000:0.0:1767318951.648675:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985045760:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318951.648714:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318951.650454:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2014 00000100:00100000:1.0:1767318951.651094:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985045632:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318951.773398:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318952.795381:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318953.819808:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318954.157037:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767318954.157128:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920533504 00000100:00100000:2.0:1767318954.157138:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920533504 00000100:00100000:2.0:1767318954.157245:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4328 00000100:00100000:2.0:1767318954.157258:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920533504:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767318954.157342:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920533504:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318954.157349:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4328 00000100:00100000:1.0:1767318954.157517:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318954.157838:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920533376 00000100:00100000:3.0:1767318954.157843:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920533376 00000100:00100000:3.0:1767318954.157883:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4707 00000100:00100000:3.0:1767318954.157889:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920533376:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318954.157909:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318954.157939:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920533376:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 50us (425us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318954.157946:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4707 00000100:00080000:0.0:1767318954.158516:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318954.203373:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318954.203384:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318954.203399:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318954.203404:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318954.203408:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318954.203416:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4197 00000100:00080000:3.0:1767318954.203421:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318954.203428:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318954.203470:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beaa00 x1853163985045888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318954.203494:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318954.203497:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318954.203500:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318954.203503:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4197 00000100:00080000:3.0:1767318954.203506:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318954.203509:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318954.203521:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8380 x1853163985046016/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318954.203563:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318954.203567:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318954.203569:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318954.203573:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4197 00000100:00080000:3.0:1767318954.203575:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318954.203579:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318954.203590:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8e00 x1853163985046272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318954.203605:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318954.203608:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318954.203610:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318954.203613:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4197 00000100:00080000:3.0:1767318954.203615:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318954.203618:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318954.203642:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb480 x1853163985046528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318954.203747:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985046656:0@lo:400:kworker.0 00000100:00100000:1.0:1767318954.203812:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318954.203819:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985046400:0@lo:400:kworker.0 00000100:00100000:2.0:1767318954.203857:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318954.203894:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318954.203897:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318954.203904:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985046784:0@lo:400:kworker.0 00000100:00100000:2.0:1767318954.203908:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985046144:0@lo:400:kworker.0 00000100:00100000:1.0:1767318954.203942:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318954.203948:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318954.203983:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046400 00000100:00100000:1.0:1767318954.203991:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046400 00000100:00100000:2.0:1767318954.204003:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046144 00000100:00100000:2.0:1767318954.204018:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046144 00000100:00100000:1.0:1767318954.204046:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4329 00000100:00100000:1.0:1767318954.204059:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985046400:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318954.204060:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4330 00000100:00100000:2.0:1767318954.204073:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985046144:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318954.204123:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985046400:12345-0@lo:400:kworker.0 Request processed in 67us (264us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318954.204127:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985046144:12345-0@lo:400:kworker.0 Request processed in 55us (180us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318954.204136:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4329 00000100:00100000:3.0:1767318954.204337:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046656 00000100:00100000:3.0:1767318954.204342:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046656 00000100:00100000:3.0:1767318954.204381:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046784 00000100:00100000:3.0:1767318954.204384:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046784 00000100:00100000:3.0:1767318954.204412:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2861 00000100:00100000:3.0:1767318954.204419:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985046656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318954.204433:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985045888:0@lo:38:kworker.0 00000100:00100000:3.0:1767318954.204475:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985046656:12345-0@lo:400:kworker.0 Request processed in 56us (662us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318954.204476:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318954.204487:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2861 00000100:00100000:3.0:1767318954.204504:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2862 00000100:00100000:3.0:1767318954.204511:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985046784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318954.204514:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985046016:0@lo:38:kworker.0 00000100:00100000:0.0:1767318954.204550:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318954.204555:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985046784:12345-0@lo:400:kworker.0 Request processed in 43us (611us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318954.204562:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2862 00000100:00100000:0.0:1767318954.204582:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985046272:0@lo:38:kworker.0 00000100:00100000:3.0:1767318954.204588:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046016 00000100:00100000:3.0:1767318954.204593:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046016 00000100:00100000:0.0:1767318954.204603:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318954.204620:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1667 00000100:00100000:3.0:1767318954.204626:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985046016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318954.204644:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318954.204695:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985046016:12345-0@lo:38:kworker.0 Request processed in 68us (144us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318954.204703:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1667 00000100:00100000:2.0:1767318954.204748:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4330 00000100:00100000:3.0:1767318954.204752:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985046144:0@lo:400:kworker.0 00000100:00100000:2.0:1767318954.204768:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985045888 00000100:00100000:3.0:1767318954.204768:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985046400:0@lo:400:kworker.0 00000100:00100000:2.0:1767318954.204774:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985045888 00000100:00100000:2.0:1767318954.204792:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046272 00000100:00100000:2.0:1767318954.204795:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046272 00000100:00100000:2.0:1767318954.204815:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4708 00000100:00080000:3.0:1767318954.204819:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318954.204821:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985045888:12345-0@lo:38:kworker.0 00000100:00080000:3.0:1767318954.204825:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318954.204829:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:2.0:1767318954.204835:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318954.204860:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4709 00000100:00100000:2.0:1767318954.204865:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985045888:12345-0@lo:38:kworker.0 Request processed in 41us (387us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318954.204871:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985046272:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767318954.204878:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4708 00010000:00020000:3.0:1767318954.204884:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318954.204915:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985046272:12345-0@lo:38:kworker.0 Request processed in 46us (312us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318954.204921:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4709 00000100:00100000:1.0:1767318954.205197:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985046784:0@lo:400:kworker.0 00000100:00100000:1.0:1767318954.205207:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985046656:0@lo:400:kworker.0 00000100:00100000:0.0:1767318954.205567:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985046528:0@lo:38:kworker.0 00000100:00100000:0.0:1767318954.205599:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318954.205739:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318954.205745:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318954.205748:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985045888:0@lo:38:kworker.0 00000100:00080000:0.0:1767318954.205758:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318954.205761:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318954.205763:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985046016:0@lo:38:kworker.0 00000100:00080000:0.0:1767318954.205772:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318954.205775:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318954.205778:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985046272:0@lo:38:kworker.0 00000100:00100000:1.0:1767318954.205915:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046528 00000100:00100000:1.0:1767318954.205921:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046528 00000100:00100000:1.0:1767318954.205958:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4710 00000100:00100000:1.0:1767318954.205974:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985046528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318954.205985:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318954.206427:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985046528:12345-0@lo:38:kworker.0 Request processed in 460us (825us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318954.206434:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4710 00000100:00080000:0.0:1767318954.206761:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318954.206766:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318954.206769:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985046528:0@lo:38:kworker.0 00010000:00010000:0.0:1767318954.843579:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318955.868044:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318956.763430:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985047040:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318956.763453:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ade680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985046912:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318956.763525:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318956.763667:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318956.763740:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047040 00000100:00100000:1.0:1767318956.763747:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047040 00000100:00100000:1.0:1767318956.763805:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985046912 00000100:00100000:1.0:1767318956.763809:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985046912 00000100:00100000:1.0:1767318956.763832:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2016 00000100:00100000:1.0:1767318956.763839:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985047040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318956.763870:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318956.763931:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985047040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (419us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318956.763939:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2016 00000100:00100000:1.0:1767318956.763955:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2017 00000100:00100000:1.0:1767318956.763961:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985046912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318956.763977:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767318956.764047:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318956.764051:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318956.768101:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985047040:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318956.770068:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985046912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 6098us (6392us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318956.770093:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2017 00000100:00100000:0.0:1767318956.770238:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ade680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985046912:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318956.893373:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318957.915548:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318958.941028:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318959.277401:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318959.277447:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318959.278521:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920533760 00000100:00100000:0.0:1767318959.280830:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920533760 00000100:00080000:2.0:1767318959.280924:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318959.280941:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920533632 00000100:00100000:0.0:1767318959.280945:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920533632 00000100:00100000:0.0:1767318959.280966:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4331 00000100:00100000:2.0:1767318959.280972:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4711 00000100:00100000:0.0:1767318959.280977:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920533760:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767318959.280978:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920533632:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767318959.280999:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318959.281194:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920533760:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 219us (3802us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318959.281203:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4331 00000100:00100000:2.0:1767318959.283125:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920533632:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 2134us (5666us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318959.283152:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4711 00000100:00080000:3.0:1767318959.323349:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318959.323364:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318959.323367:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318959.323373:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318959.323379:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318959.323391:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4202 00000100:00080000:3.0:1767318959.323397:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318959.323406:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318959.323459:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bebb80 x1853163985047168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318959.323477:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318959.323481:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318959.323489:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318959.323493:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4202 00000100:00080000:3.0:1767318959.323496:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318959.323500:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318959.323512:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9c00 x1853163985047296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318959.323560:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318959.323565:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318959.323566:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318959.323570:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4202 00000100:00080000:3.0:1767318959.323572:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318959.323575:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318959.323595:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bea300 x1853163985047552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318959.323609:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318959.323612:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318959.323614:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318959.323617:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4202 00000100:00080000:3.0:1767318959.323619:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318959.323622:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318959.323632:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb800 x1853163985047808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318959.323849:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985047424:0@lo:400:kworker.0 00000100:00100000:2.0:1767318959.323849:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985047680:0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.323921:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985048064:0@lo:400:kworker.0 00000100:00100000:1.0:1767318959.324921:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318959.326954:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318959.328759:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985047168:0@lo:38:kworker.0 00000100:00100000:0.0:1767318959.328814:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318959.329633:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318959.329773:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985048064 00000100:00100000:1.0:1767318959.329782:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985048064 00000100:00100000:1.0:1767318959.329841:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2863 00000100:00100000:1.0:1767318959.329850:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985048064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318959.329923:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985048064:12345-0@lo:400:kworker.0 Request processed in 74us (1716us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318959.329941:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2863 00000100:00100000:1.0:1767318959.329968:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047424 00000100:00100000:1.0:1767318959.329972:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047424 00000100:00100000:1.0:1767318959.329998:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047680 00000100:00100000:1.0:1767318959.330000:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047680 00000100:00100000:1.0:1767318959.330017:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047168 00000100:00100000:1.0:1767318959.330020:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047168 00000100:00100000:1.0:1767318959.330034:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4332 00000100:00100000:1.0:1767318959.330042:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985047424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318959.330079:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985047424:12345-0@lo:400:kworker.0 Request processed in 38us (5161us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318959.330084:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4332 00000100:00100000:1.0:1767318959.330096:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4333 00000100:00100000:1.0:1767318959.330101:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985047680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318959.330127:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985047680:12345-0@lo:400:kworker.0 Request processed in 26us (3182us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318959.330133:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4333 00000100:00100000:1.0:1767318959.330143:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4712 00000100:00100000:1.0:1767318959.330148:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ade680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985047168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318959.330186:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318959.330231:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ade680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985047168:12345-0@lo:38:kworker.0 Request processed in 81us (1417us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318959.330236:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4712 00000100:00100000:1.0:1767318959.330274:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985047424:0@lo:400:kworker.0 00000100:00080000:1.0:1767318959.330411:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318959.330416:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318959.330443:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985047936:0@lo:400:kworker.0 00000100:00100000:1.0:1767318959.330477:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318959.330507:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047936 00000100:00100000:1.0:1767318959.330510:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047936 00000100:00100000:1.0:1767318959.330534:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2864 00000100:00100000:1.0:1767318959.330539:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985047936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318959.330569:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985047936:12345-0@lo:400:kworker.0 Request processed in 30us (92us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318959.330574:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2864 00000100:00100000:1.0:1767318959.330612:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985047936:0@lo:400:kworker.0 00000100:00080000:1.0:1767318959.330626:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318959.335029:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985047680:0@lo:400:kworker.0 00000100:00080000:0.0:1767318959.337666:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318959.337679:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318959.337685:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985047168:0@lo:38:kworker.0 00000100:00100000:0.0:1767318959.337697:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985047296:0@lo:38:kworker.0 00000100:00100000:0.0:1767318959.337759:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318959.337808:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985047552:0@lo:38:kworker.0 00000100:00100000:0.0:1767318959.337834:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318959.337858:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985047808:0@lo:38:kworker.0 00000100:00100000:0.0:1767318959.337891:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318959.337962:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047552 00000100:00100000:0.0:1767318959.337970:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047552 00000100:00100000:0.0:1767318959.338002:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047808 00000100:00100000:0.0:1767318959.338005:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047808 00000100:00100000:0.0:1767318959.338023:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4713 00000100:00100000:0.0:1767318959.338029:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6addc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985047552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318959.338051:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318959.338089:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6addc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985047552:12345-0@lo:38:kworker.0 Request processed in 59us (254us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318959.338095:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4713 00000100:00100000:0.0:1767318959.338109:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4714 00000100:00100000:0.0:1767318959.338112:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6add180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985047808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318959.338121:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318959.338139:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6add180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985047808:12345-0@lo:38:kworker.0 Request processed in 25us (248us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318959.338144:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4714 00000100:00080000:0.0:1767318959.338192:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318959.338196:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318959.338200:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985047552:0@lo:38:kworker.0 00000100:00080000:0.0:1767318959.338544:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318959.338548:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318959.338551:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985047808:0@lo:38:kworker.0 00000100:00100000:2.0:1767318959.340527:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985047296 00000100:00100000:2.0:1767318959.340542:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985047296 00000100:00100000:2.0:1767318959.340599:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1668 00000100:00100000:2.0:1767318959.340607:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ade300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985047296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318959.340632:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318959.340726:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ade300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985047296:12345-0@lo:38:kworker.0 Request processed in 119us (2967us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318959.340734:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1668 00000100:00080000:1.0:1767318959.340861:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318959.340875:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318959.340880:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985047296:0@lo:38:kworker.0 00000100:00100000:3.0:1767318959.341440:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985048064:0@lo:400:kworker.0 00010000:00010000:2.0:1767318959.963536:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318960.987398:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318961.886390:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1767318961.886418:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985048320:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318961.886488:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318961.886529:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6add500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985048192:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318961.886550:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985048320 00000100:00100000:2.0:1767318961.886559:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985048320 00000100:00100000:0.0:1767318961.886562:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318961.886612:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318961.886629:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985048192 00000100:00100000:2.0:1767318961.886633:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985048192 00000100:00100000:0.0:1767318961.886635:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2018 00000100:00100000:0.0:1767318961.886644:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985048320:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318961.886656:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2019 00000100:00100000:2.0:1767318961.886663:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6add880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985048192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318961.886680:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:2.0:1767318961.886680:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767318961.886827:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318961.886877:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985048320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 234us (387us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318961.886885:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2018 00000100:00100000:2.0:1767318961.887146:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6add880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985048192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 483us (580us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318961.887152:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2019 00000100:00100000:1.0:1767318961.892331:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985048320:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318961.892372:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6add500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985048192:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318962.015013:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318963.040520:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318963.373645:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767318963.373690:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920533888 offset 0 length 520: 4 00000100:00100000:0.0:1767318963.373736:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767318963.373844:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534144 00000100:00100000:2.0:1767318963.373855:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534144 00000100:00100000:2.0:1767318963.373936:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534016 00000100:00100000:2.0:1767318963.373940:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534016 00000100:00100000:2.0:1767318963.373962:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4334 00000100:00080000:3.0:1767318963.373972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318963.373975:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6addf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920534144:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767318963.374071:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6addf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920534144:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (437us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318963.374080:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4334 00000100:00100000:2.0:1767318963.374108:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4715 00000100:00100000:2.0:1767318963.374113:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6add500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920534016:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767318963.374134:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318963.374194:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6add500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920534016:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (425us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318963.374202:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4715 00000100:00100000:0.0:1767318963.419491:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318938/real 1767318938] req@ffff8ce19e9c6a00 x1853163985041920/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318963 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318963.419535:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19e9c6a00 x1853163985041920/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318963 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767318963.419554:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318963.419579:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318963.419584:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985041920:0@lo:250:kworker.0 00000100:00080000:3.0:1767318963.419622:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318963.419628:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767318963.419631:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318963.419637:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4187 00000100:00080000:3.0:1767318963.419641:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318963.419649:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318963.419681:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beaa00 x1853163985048448/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318963.419712:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318963.419716:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318963.419718:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318963.419722:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4207 00000100:00080000:3.0:1767318963.419725:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318963.419729:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318963.419744:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bead80 x1853163985048576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318963.419754:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318963.419757:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318963.419759:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318963.419762:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4207 00000100:00080000:3.0:1767318963.419765:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318963.419768:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318963.419778:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8000 x1853163985048704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318963.419833:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318963.419837:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318963.419840:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318963.419844:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4207 00000100:00080000:3.0:1767318963.419847:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767318963.419849:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767318963.419859:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318963.419865:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985048832:0@lo:400:kworker.0 00000100:00100000:3.0:1767318963.419875:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8e00 x1853163985048960/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318963.419904:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318963.419907:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318963.419909:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318963.419912:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4207 00000100:00080000:3.0:1767318963.419914:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318963.419918:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318963.419929:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb800 x1853163985049216/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318963.419937:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318963.419975:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985049344:0@lo:400:kworker.0 00000100:00100000:1.0:1767318963.420003:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985049472:0@lo:400:kworker.0 00000100:00100000:3.0:1767318963.420013:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318963.420035:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318963.420041:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318963.420049:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985049088:0@lo:400:kworker.0 00000100:00100000:1.0:1767318963.420076:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049344 00000100:00100000:1.0:1767318963.420095:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049344 00000100:00100000:1.0:1767318963.420149:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049472 00000100:00100000:1.0:1767318963.420153:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049472 00000100:00100000:0.0:1767318963.420186:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985048448:0@lo:250:kworker.0 00000100:00100000:1.0:1767318963.420203:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2865 00000100:00100000:1.0:1767318963.420213:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985049344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318963.420360:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2866 00000100:00100000:2.0:1767318963.420381:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985049472:12345-0@lo:400:kworker.0 00000400:00000100:0.0:1767318963.424485:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985048448 offset 0 length 520: 4 00000100:00100000:0.0:1767318963.425683:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985048576:0@lo:38:kworker.0 00000100:00100000:0.0:1767318963.426273:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318963.426307:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985048704:0@lo:38:kworker.0 00000100:00100000:0.0:1767318963.426336:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318963.426354:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985048960:0@lo:38:kworker.0 00000100:00100000:0.0:1767318963.426374:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318963.426394:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985049216:0@lo:38:kworker.0 00000100:00100000:0.0:1767318963.426413:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318963.426463:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985048704 00000100:00100000:0.0:1767318963.426471:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985048704 00000100:00100000:0.0:1767318963.426943:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1669 00000100:00100000:0.0:1767318963.426951:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985048704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318963.426973:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318963.427024:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985048704:12345-0@lo:38:kworker.0 Request processed in 71us (686us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318963.427031:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1669 00000100:00100000:0.0:1767318963.427057:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985048832 00000100:00100000:0.0:1767318963.427062:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985048832 00000100:00100000:0.0:1767318963.427284:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985048576 00000100:00100000:0.0:1767318963.427288:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985048576 00000100:00100000:0.0:1767318963.427312:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985048960 00000100:00100000:0.0:1767318963.427315:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985048960 00000100:00100000:0.0:1767318963.427322:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049216 00000100:00100000:0.0:1767318963.427325:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049216 00000100:00100000:0.0:1767318963.427334:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4335 00000100:00100000:0.0:1767318963.427342:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985048832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318963.427390:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985048832:12345-0@lo:400:kworker.0 Request processed in 50us (7457us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318963.427400:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4335 00000100:00100000:0.0:1767318963.427413:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4716 00000100:00100000:0.0:1767318963.427417:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985048576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318963.427427:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318963.427452:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985048576:12345-0@lo:38:kworker.0 Request processed in 34us (1183us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318963.427458:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4716 00000100:00100000:0.0:1767318963.427469:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4717 00000100:00100000:0.0:1767318963.427473:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985048960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318963.427481:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318963.427932:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985048960:12345-0@lo:38:kworker.0 Request processed in 26us (1127us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318963.427943:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4717 00000100:00100000:0.0:1767318963.427957:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4718 00000100:00100000:0.0:1767318963.427961:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985049216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318963.427972:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318963.428008:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985049216:12345-0@lo:38:kworker.0 Request processed in 45us (1593us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318963.428013:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4718 00000100:00100000:0.0:1767318963.428122:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985048832:0@lo:400:kworker.0 00000100:00080000:0.0:1767318963.428185:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318963.428190:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318963.428194:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985048576:0@lo:38:kworker.0 00000100:00080000:0.0:1767318963.428207:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318963.428211:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318963.428214:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985048704:0@lo:38:kworker.0 00000100:00080000:0.0:1767318963.428221:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318963.428224:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318963.428227:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985048960:0@lo:38:kworker.0 00000100:00080000:0.0:1767318963.428234:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318963.428237:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318963.428240:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985049216:0@lo:38:kworker.0 00000100:00080000:0.0:1767318963.428293:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318963.428299:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318963.428302:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318963.437352:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985049344:12345-0@lo:400:kworker.0 Request processed in 17129us (17327us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318963.437367:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2865 00000100:00100000:3.0:1767318963.437774:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318963.438106:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985049344:0@lo:400:kworker.0 00000100:00100000:0.0:1767318963.439595:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049088 00000100:00100000:0.0:1767318963.439607:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049088 00000100:00100000:0.0:1767318963.439748:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4336 00000100:00100000:0.0:1767318963.439758:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985049088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318963.439870:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985049088:12345-0@lo:400:kworker.0 Request processed in 113us (2096us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318963.439877:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4336 00000100:00080000:3.0:1767318963.446670:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318963.446823:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985049088:0@lo:400:kworker.0 00000100:00100000:2.0:1767318963.453625:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985049472:12345-0@lo:400:kworker.0 Request processed in 33245us (33578us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318963.453642:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2866 00000100:00100000:0.0:1767318963.457490:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985049472:0@lo:400:kworker.0 00010000:00010000:1.0:1767318964.060448:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318965.085221:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318966.108828:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318967.007346:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318967.007372:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985049600:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318967.007464:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318967.007690:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049600 00000100:00100000:0.0:1767318967.007700:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049600 00000100:00100000:0.0:1767318967.007771:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2020 00000100:00100000:0.0:1767318967.007781:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985049600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318967.007814:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318967.007861:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985049600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (396us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318967.007868:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2020 00000100:00080000:1.0:1767318967.008010:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318967.008046:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985049600:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318967.012023:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985049728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318967.012101:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318967.012351:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049728 00000100:00100000:1.0:1767318967.012361:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049728 00000100:00100000:1.0:1767318967.012453:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2021 00000100:00100000:1.0:1767318967.012464:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985049728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318967.012502:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318967.012558:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985049728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (457us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318967.012566:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2021 00000100:00080000:3.0:1767318967.012639:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318967.012770:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985049728:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767318967.131486:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318968.155458:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318968.492569:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318968.492918:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318968.500539:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534400 00000100:00100000:0.0:1767318968.500556:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534400 00000100:00100000:0.0:1767318968.500644:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534272 00000100:00100000:0.0:1767318968.500648:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534272 00000100:00080000:3.0:1767318968.500650:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318968.500677:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4337 00000100:00100000:0.0:1767318968.500691:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920534400:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767318968.500693:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4719 00000100:00100000:3.0:1767318968.500713:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920534272:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318968.500736:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318968.501592:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920534400:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 900us (9027us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318968.501604:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4337 00000100:00100000:3.0:1767318968.508189:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920534272:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 7451us (15247us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318968.508452:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4719 00000100:00080000:0.0:1767318968.546437:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.546449:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318968.546453:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.546457:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.546462:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.546470:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.546475:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.546481:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318968.546525:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985049856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318968.546542:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.546546:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.546548:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.546551:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.546555:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.546558:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318968.546574:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74700 x1853163985049984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318968.546629:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.546632:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.546634:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.546638:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.546640:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.546643:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318968.546653:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76a00 x1853163985050240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318968.546684:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.546687:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.546689:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.546692:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.546695:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.546698:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318968.546708:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163985050496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318968.546749:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985050624:0@lo:400:kworker.0 00000100:00100000:0.0:1767318968.546809:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318968.546882:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318968.546891:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985050112:0@lo:400:kworker.0 00000100:00100000:0.0:1767318968.546893:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985050368:0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.546939:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318968.546973:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985050624 00000100:00100000:3.0:1767318968.546981:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985050112 00000100:00100000:0.0:1767318968.546983:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985050624 00000100:00100000:3.0:1767318968.546988:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985050112 00000100:00100000:0.0:1767318968.547035:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2867 00000100:00100000:3.0:1767318968.547040:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4338 00000100:00100000:0.0:1767318968.547046:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985050624:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.547053:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985050112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.547104:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985050112:12345-0@lo:400:kworker.0 Request processed in 55us (164us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318968.547111:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985050624:12345-0@lo:400:kworker.0 Request processed in 67us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318968.547114:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4338 00000100:00100000:0.0:1767318968.547125:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2867 00000100:00080000:3.0:1767318968.547144:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318968.547153:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.547217:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985050112:0@lo:400:kworker.0 00000100:00100000:0.0:1767318968.547222:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985050624:0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.547240:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318968.547253:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985050368 00000100:00100000:3.0:1767318968.547258:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985050368 00000100:00100000:3.0:1767318968.547325:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4339 00000100:00100000:3.0:1767318968.547334:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985050368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.547382:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985050368:12345-0@lo:400:kworker.0 Request processed in 48us (229us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318968.547389:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4339 00000100:00100000:3.0:1767318968.547423:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985050752:0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.547447:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.547540:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985050752 00000100:00100000:3.0:1767318968.547545:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985050752 00000100:00100000:3.0:1767318968.547569:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2868 00000100:00100000:3.0:1767318968.547575:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985050752:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.547604:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985050752:12345-0@lo:400:kworker.0 Request processed in 29us (156us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318968.547611:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2868 00000100:00100000:3.0:1767318968.547637:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985050752:0@lo:400:kworker.0 00000100:00100000:1.0:1767318968.547968:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985050368:0@lo:400:kworker.0 00000100:00080000:0.0:1767318968.548097:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318968.548130:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985049856:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.548160:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.548217:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985049984:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.548239:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.548268:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985050240:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.548288:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318968.548360:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049984 00000100:00100000:2.0:1767318968.548365:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049984 00000100:00100000:3.0:1767318968.548401:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985050496:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.548434:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318968.548472:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1670 00000100:00100000:2.0:1767318968.548477:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985049984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318968.548496:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318968.548530:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985049984:12345-0@lo:38:kworker.0 Request processed in 52us (290us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318968.548538:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1670 00000100:00080000:3.0:1767318968.552525:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.552537:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.552544:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985049984:0@lo:38:kworker.0 00000100:00100000:1.0:1767318968.554385:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985049856 00000100:00100000:1.0:1767318968.554400:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985049856 00000100:00100000:1.0:1767318968.554465:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985050240 00000100:00100000:1.0:1767318968.554468:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985050240 00000100:00100000:1.0:1767318968.554487:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985050496 00000100:00100000:1.0:1767318968.554490:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985050496 00000100:00100000:1.0:1767318968.554510:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4720 00000100:00100000:1.0:1767318968.554518:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985049856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318968.554543:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318968.554621:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985049856:12345-0@lo:38:kworker.0 Request processed in 102us (6458us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318968.554628:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4720 00000100:00100000:1.0:1767318968.554644:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4721 00000100:00100000:1.0:1767318968.554649:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985050240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318968.554659:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318968.554685:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985050240:12345-0@lo:38:kworker.0 Request processed in 34us (6396us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318968.554691:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4721 00000100:00100000:1.0:1767318968.554702:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4722 00000100:00100000:1.0:1767318968.554706:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985050496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318968.554714:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318968.554737:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985050496:12345-0@lo:38:kworker.0 Request processed in 29us (6302us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318968.554742:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4722 00000100:00080000:3.0:1767318968.555803:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.555812:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.555817:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985049856:0@lo:38:kworker.0 00000100:00080000:3.0:1767318968.555828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.555832:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.555835:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985050240:0@lo:38:kworker.0 00000100:00080000:3.0:1767318968.555845:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.555848:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.555851:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985050496:0@lo:38:kworker.0 00010000:00010000:1.0:1767318969.179571:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318970.203410:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318971.227456:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318972.123494:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985050880:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318972.123572:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318972.123943:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985050880 00000100:00100000:2.0:1767318972.123952:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985050880 00000100:00100000:2.0:1767318972.124021:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2022 00000100:00100000:2.0:1767318972.124034:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985050880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318972.124065:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767318972.124111:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985050880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (539us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318972.124118:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2022 00000100:00080000:0.0:1767318972.124525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318972.124567:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985050880:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318972.125271:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318972.125285:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985051008:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318972.125334:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318972.125429:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051008 00000100:00100000:0.0:1767318972.125437:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051008 00000100:00100000:0.0:1767318972.125485:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2023 00000100:00100000:0.0:1767318972.125492:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985051008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318972.125513:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318972.125555:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985051008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (220us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318972.125562:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2023 00000100:00100000:0.0:1767318972.125600:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985051008:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318972.125823:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767318972.251379:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318973.278655:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318973.611105:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318973.611159:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767318973.612651:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534528 00000100:00100000:2.0:1767318973.612801:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534528 00000100:00100000:2.0:1767318973.612853:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534656 00000100:00100000:2.0:1767318973.612858:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534656 00000100:00100000:2.0:1767318973.612912:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4340 00000100:00100000:2.0:1767318973.612925:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920534656:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767318973.613000:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920534656:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (1840us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318973.613009:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4340 00000100:00100000:2.0:1767318973.613028:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4723 00000100:00100000:2.0:1767318973.613033:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920534528:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767318973.613053:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318973.613078:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920534528:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (1984us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318973.613084:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4723 00000100:00080000:1.0:1767318973.614015:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318973.660446:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.660455:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318973.660459:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.660463:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.660467:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318973.660475:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.660479:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318973.660485:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318973.660520:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2680 x1853163985051136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318973.660546:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.660549:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.660551:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318973.660554:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.660556:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318973.660559:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318973.660569:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0380 x1853163985051264/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318973.660859:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985051392:0@lo:400:kworker.0 00000100:00100000:3.0:1767318973.660920:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.661016:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985051136:0@lo:38:kworker.0 00000100:00100000:0.0:1767318973.661044:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.661067:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985051264:0@lo:38:kworker.0 00000100:00100000:0.0:1767318973.661090:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.661138:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051264 00000100:00100000:0.0:1767318973.661145:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051264 00000100:00100000:0.0:1767318973.661198:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1671 00000100:00100000:0.0:1767318973.661205:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985051264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318973.661226:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318973.661265:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985051264:12345-0@lo:38:kworker.0 Request processed in 58us (173us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318973.661271:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1671 00000100:00080000:0.0:1767318973.661306:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318973.661321:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318973.661326:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985051264:0@lo:38:kworker.0 00000100:00100000:1.0:1767318973.661404:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051392 00000100:00100000:1.0:1767318973.661408:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051392 00000100:00100000:1.0:1767318973.661451:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051136 00000100:00100000:1.0:1767318973.661454:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051136 00000100:00100000:1.0:1767318973.661467:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4341 00000100:00100000:1.0:1767318973.661473:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985051392:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318973.661536:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318973.661561:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4724 00000100:00100000:0.0:1767318973.661566:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985051136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318973.661576:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318973.661612:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985051136:12345-0@lo:38:kworker.0 Request processed in 45us (567us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318973.661618:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4724 00000100:00080000:0.0:1767318973.661644:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318973.661647:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318973.661651:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985051136:0@lo:38:kworker.0 00000100:00080000:2.0:1767318973.666686:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.666698:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.666702:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318973.666711:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.666715:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318973.666722:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318973.666764:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985051520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318973.666804:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.666808:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.666810:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318973.666813:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.666815:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318973.666819:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318973.666830:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2a00 x1853163985051776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318973.667105:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985051904:0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667190:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.667236:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318973.667243:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985051648:0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667267:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.667294:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051648 00000100:00100000:0.0:1767318973.667301:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051648 00000100:00100000:0.0:1767318973.667368:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4342 00000100:00100000:0.0:1767318973.667377:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985051648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667425:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985051648:12345-0@lo:400:kworker.0 Request processed in 49us (156us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318973.667432:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4342 00000100:00080000:0.0:1767318973.667455:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318973.667489:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985051648:0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667529:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985052032:0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667553:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.667584:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051904 00000100:00100000:0.0:1767318973.667588:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051904 00000100:00100000:0.0:1767318973.667628:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052032 00000100:00100000:0.0:1767318973.667631:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052032 00000100:00100000:0.0:1767318973.667650:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2869 00000100:00100000:0.0:1767318973.667664:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985051904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667697:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985051904:12345-0@lo:400:kworker.0 Request processed in 41us (506us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318973.667703:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2869 00000100:00100000:0.0:1767318973.667715:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2870 00000100:00100000:0.0:1767318973.667722:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985052032:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667751:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985052032:12345-0@lo:400:kworker.0 Request processed in 30us (198us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318973.667757:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2870 00000100:00100000:0.0:1767318973.667784:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985052032:0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667800:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985051904:0@lo:400:kworker.0 00000100:00100000:0.0:1767318973.667828:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985051520:0@lo:38:kworker.0 00000100:00100000:0.0:1767318973.667852:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.667864:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985051776:0@lo:38:kworker.0 00000100:00100000:0.0:1767318973.667879:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318973.667897:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051520 00000100:00100000:0.0:1767318973.667900:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051520 00000100:00100000:0.0:1767318973.667913:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985051776 00000100:00100000:0.0:1767318973.667915:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985051776 00000100:00100000:0.0:1767318973.667927:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4725 00000100:00100000:0.0:1767318973.667931:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985051520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318973.667948:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318973.667972:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985051520:12345-0@lo:38:kworker.0 Request processed in 40us (120us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318973.667978:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4725 00000100:00100000:0.0:1767318973.667987:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4726 00000100:00100000:0.0:1767318973.667991:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985051776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318973.667999:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318973.668014:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985051776:12345-0@lo:38:kworker.0 Request processed in 22us (135us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318973.668020:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4726 00000100:00080000:0.0:1767318973.668045:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318973.668050:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318973.668053:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985051520:0@lo:38:kworker.0 00000100:00080000:0.0:1767318973.668060:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318973.668063:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318973.668065:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985051776:0@lo:38:kworker.0 00000100:00100000:1.0:1767318973.668417:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985051392:12345-0@lo:400:kworker.0 Request processed in 6941us (7494us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318973.668425:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4341 00000100:00100000:3.0:1767318973.668904:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985051392:0@lo:400:kworker.0 00010000:00010000:1.0:1767318974.299396:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318975.331782:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318976.347463:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318977.243451:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318977.243473:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985052160:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318977.243539:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318977.243807:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985052288:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318977.243841:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318977.244021:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052160 00000100:00100000:3.0:1767318977.244029:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052160 00000100:00100000:3.0:1767318977.244088:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052288 00000100:00100000:3.0:1767318977.244092:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052288 00000100:00100000:3.0:1767318977.244116:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2024 00000100:00100000:3.0:1767318977.244124:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985052160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318977.244153:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318977.244218:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985052160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (678us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318977.244225:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2024 00000100:00100000:3.0:1767318977.244255:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2025 00000100:00100000:3.0:1767318977.244263:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985052288:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318977.244278:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318977.244305:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985052288:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (464us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318977.244311:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2025 00000100:00080000:0.0:1767318977.244374:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318977.244379:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318977.244422:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985052160:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318977.244646:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985052288:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767318977.373969:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318978.395911:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318978.734071:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318978.734239:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318978.734420:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534912 00000100:00100000:0.0:1767318978.734438:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534912 00000100:00100000:0.0:1767318978.734530:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920534784 00000100:00100000:0.0:1767318978.734535:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920534784 00000100:00100000:0.0:1767318978.734557:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4343 00000100:00100000:0.0:1767318978.734583:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920534912:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767318978.734671:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920534912:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (614us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318978.734679:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4343 00000100:00100000:0.0:1767318978.734702:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4727 00000100:00100000:0.0:1767318978.734709:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920534784:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767318978.734733:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318978.734771:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920534784:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 59us (529us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318978.734779:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4727 00000100:00080000:3.0:1767318978.735066:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318978.780939:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318978.780953:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318978.780956:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318978.780961:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318978.780966:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318978.780978:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4221 00000100:00080000:2.0:1767318978.780984:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318978.780993:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318978.781042:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3800 x1853163985052416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318978.781074:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318978.781079:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318978.781081:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318978.781085:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4221 00000100:00080000:2.0:1767318978.781088:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318978.781092:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318978.781109:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985052544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318978.781238:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318978.781243:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318978.781245:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318978.781251:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4221 00000100:00080000:2.0:1767318978.781254:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318978.781258:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318978.781278:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1c00 x1853163985052800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318978.781287:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985052672:0@lo:400:kworker.0 00000100:00080000:2.0:1767318978.781370:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767318978.781377:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318978.781383:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318978.781390:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318978.781397:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4221 00000100:00080000:2.0:1767318978.781404:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318978.781413:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318978.781449:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2a00 x1853163985053056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318978.781577:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318978.781591:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985053312:0@lo:400:kworker.0 00000100:00100000:1.0:1767318978.781629:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318978.781694:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985053184:0@lo:400:kworker.0 00000100:00100000:2.0:1767318978.781799:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318978.781856:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318978.781865:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985052928:0@lo:400:kworker.0 00000100:00100000:1.0:1767318978.781873:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053312 00000100:00100000:1.0:1767318978.781883:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053312 00000100:00100000:2.0:1767318978.781894:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318978.781939:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052672 00000100:00100000:2.0:1767318978.781946:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052672 00000100:00100000:1.0:1767318978.781948:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053184 00000100:00100000:1.0:1767318978.781955:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053184 00000100:00100000:2.0:1767318978.782045:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052928 00000100:00080000:3.0:1767318978.782046:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318978.782058:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052928 00000100:00100000:3.0:1767318978.782089:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4344 00000100:00100000:3.0:1767318978.782102:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985052672:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318978.782120:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2871 00000100:00100000:2.0:1767318978.782133:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4345 00000100:00100000:1.0:1767318978.782136:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985053312:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318978.782157:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985052928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318978.782242:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985052672:12345-0@lo:400:kworker.0 Request processed in 140us (864us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318978.782254:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4344 00000100:00100000:1.0:1767318978.782263:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985053312:12345-0@lo:400:kworker.0 Request processed in 130us (629us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318978.782273:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2871 00000100:00080000:3.0:1767318978.782288:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318978.782314:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2872 00000100:00100000:1.0:1767318978.782326:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985053184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318978.782339:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985053312:0@lo:400:kworker.0 00000100:00100000:3.0:1767318978.782360:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985052672:0@lo:400:kworker.0 00000100:00100000:1.0:1767318978.782378:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985053184:12345-0@lo:400:kworker.0 Request processed in 52us (578us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318978.782388:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2872 00000100:00100000:3.0:1767318978.782454:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985052928:0@lo:400:kworker.0 00000100:00100000:2.0:1767318978.782455:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985052928:12345-0@lo:400:kworker.0 Request processed in 175us (427us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318978.782469:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985053184:0@lo:400:kworker.0 00000100:00100000:2.0:1767318978.782476:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4345 00000100:00100000:0.0:1767318978.782737:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985052416:0@lo:38:kworker.0 00000100:00100000:0.0:1767318978.782778:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318978.782815:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985052544:0@lo:38:kworker.0 00000100:00100000:0.0:1767318978.782842:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318978.782866:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985052800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318978.782887:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318978.782907:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985053056:0@lo:38:kworker.0 00000100:00100000:0.0:1767318978.782926:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318978.782962:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052416 00000100:00100000:0.0:1767318978.782966:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052416 00000100:00100000:0.0:1767318978.782988:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052800 00000100:00100000:0.0:1767318978.782990:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052800 00000100:00100000:0.0:1767318978.783004:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053056 00000100:00100000:0.0:1767318978.783006:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053056 00000100:00100000:0.0:1767318978.783017:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4728 00000100:00100000:0.0:1767318978.783022:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985052416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318978.783045:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318978.783092:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985052416:12345-0@lo:38:kworker.0 Request processed in 68us (312us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318978.783098:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4728 00000100:00100000:0.0:1767318978.783113:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4729 00000100:00100000:0.0:1767318978.783117:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985052800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318978.783126:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318978.783151:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985052800:12345-0@lo:38:kworker.0 Request processed in 33us (263us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318978.783157:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4729 00000100:00100000:0.0:1767318978.783195:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4730 00000100:00100000:0.0:1767318978.783199:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985053056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318978.783208:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318978.783306:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318978.783313:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318978.783316:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985052416:0@lo:38:kworker.0 00000100:00080000:3.0:1767318978.783325:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318978.783329:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318978.783332:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985052800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318978.783703:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985053056:12345-0@lo:38:kworker.0 Request processed in 497us (770us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318978.783726:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4730 00000100:00080000:3.0:1767318978.783894:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318978.783899:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318978.783902:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985053056:0@lo:38:kworker.0 00000100:00100000:1.0:1767318978.784859:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985052544 00000100:00100000:1.0:1767318978.784871:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985052544 00000100:00100000:1.0:1767318978.784925:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1672 00000100:00100000:1.0:1767318978.784933:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985052544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318978.784953:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318978.785037:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985052544:12345-0@lo:38:kworker.0 Request processed in 103us (2193us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318978.785043:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1672 00000100:00080000:3.0:1767318978.785271:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318978.785285:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318978.785293:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985052544:0@lo:38:kworker.0 00010000:00010000:0.0:1767318979.420974:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318980.445269:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318981.467537:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318982.363570:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985053440:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318982.363641:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318982.363764:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053440 00000100:00100000:2.0:1767318982.363773:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053440 00000100:00100000:2.0:1767318982.363850:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2026 00000100:00100000:2.0:1767318982.363861:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985053440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318982.363889:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767318982.363936:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985053440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (293us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318982.363943:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2026 00000100:00100000:2.0:1767318982.363987:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985053440:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318982.364230:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318982.364420:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985053568:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318982.364452:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318982.364557:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053568 00000100:00100000:1.0:1767318982.364563:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053568 00000100:00100000:1.0:1767318982.364601:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2027 00000100:00100000:1.0:1767318982.364608:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985053568:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318982.364624:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318982.364663:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985053568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (210us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318982.364670:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2027 00000100:00100000:0.0:1767318982.364730:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985053568:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767318982.364811:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318982.493035:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318983.516035:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318983.856058:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318983.856595:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318983.863035:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920535168 00000100:00100000:0.0:1767318983.863053:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920535168 00000100:00080000:3.0:1767318983.863237:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318983.864696:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920535040 00000100:00100000:0.0:1767318983.864708:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920535040 00000100:00100000:1.0:1767318983.864796:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4346 00000100:00100000:1.0:1767318983.864812:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920535168:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767318983.864916:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920535168:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 109us (8871us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318983.864933:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4346 00000100:00100000:1.0:1767318983.864955:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4731 00000100:00100000:1.0:1767318983.864960:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920535040:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767318983.864980:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318983.865007:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920535040:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (8414us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318983.865014:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4731 00000100:00080000:2.0:1767318983.899316:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318983.899328:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318983.899332:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318983.899336:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318983.899341:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318983.899350:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4227 00000100:00080000:2.0:1767318983.899355:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318983.899362:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318983.899408:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2680 x1853163985053696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318983.899431:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318983.899435:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318983.899437:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318983.899440:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4227 00000100:00080000:2.0:1767318983.899442:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318983.899446:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318983.899458:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0380 x1853163985053824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318983.899547:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985053952:0@lo:400:kworker.0 00000100:00100000:0.0:1767318983.899631:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.899847:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053952 00000100:00100000:1.0:1767318983.899857:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053952 00000100:00100000:1.0:1767318983.899935:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4347 00000100:00100000:1.0:1767318983.899945:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985053952:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767318983.900009:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318983.900013:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318983.900015:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318983.900019:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4227 00000100:00080000:2.0:1767318983.900021:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318983.900025:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318983.900039:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1180 x1853163985054080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318983.900065:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318983.900069:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318983.900071:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318983.900075:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4227 00000100:00080000:2.0:1767318983.900077:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318983.900080:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318983.900096:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2a00 x1853163985054336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318983.900201:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985053696:0@lo:38:kworker.0 00000100:00100000:3.0:1767318983.900233:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318983.900261:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985053824:0@lo:38:kworker.0 00000100:00100000:3.0:1767318983.900282:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318983.900304:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985054080:0@lo:38:kworker.0 00000100:00100000:3.0:1767318983.900323:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318983.900337:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985054336:0@lo:38:kworker.0 00000100:00100000:3.0:1767318983.900354:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318983.900383:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053824 00000100:00100000:3.0:1767318983.900387:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053824 00000100:00100000:3.0:1767318983.900409:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1673 00000100:00100000:3.0:1767318983.900414:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985053824:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318983.900431:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318983.900463:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985053824:12345-0@lo:38:kworker.0 Request processed in 47us (179us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318983.900470:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1673 00000100:00100000:3.0:1767318983.900491:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985053696 00000100:00100000:3.0:1767318983.900494:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985053696 00000100:00100000:3.0:1767318983.900511:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985054080 00000100:00100000:3.0:1767318983.900514:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985054080 00000100:00100000:3.0:1767318983.900521:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985054336 00000100:00100000:3.0:1767318983.900524:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985054336 00000100:00100000:3.0:1767318983.900532:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4732 00000100:00100000:3.0:1767318983.900535:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985053696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318983.900542:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318983.900572:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985053696:12345-0@lo:38:kworker.0 Request processed in 35us (339us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318983.900576:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4732 00000100:00100000:3.0:1767318983.900583:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4733 00000100:00100000:3.0:1767318983.900585:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985054080:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318983.900590:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318983.900604:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985054080:12345-0@lo:38:kworker.0 Request processed in 18us (281us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318983.900607:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4733 00000100:00100000:3.0:1767318983.900612:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4734 00000100:00100000:3.0:1767318983.900614:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985054336:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318983.900619:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318983.900628:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985054336:12345-0@lo:38:kworker.0 Request processed in 13us (274us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318983.900632:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4734 00000100:00080000:3.0:1767318983.900659:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318983.900662:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318983.900664:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985053696:0@lo:38:kworker.0 00000100:00080000:3.0:1767318983.900670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318983.900672:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318983.900674:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985053824:0@lo:38:kworker.0 00000100:00080000:3.0:1767318983.900677:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318983.900679:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318983.900680:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985054080:0@lo:38:kworker.0 00000100:00080000:3.0:1767318983.900684:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318983.900685:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318983.900686:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985054336:0@lo:38:kworker.0 00000100:00100000:3.0:1767318983.900782:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985054592:0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.900800:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318983.900820:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985054592 00000100:00100000:3.0:1767318983.900822:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985054592 00000100:00100000:3.0:1767318983.900841:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2873 00000100:00100000:3.0:1767318983.900845:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985054592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.900866:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985054592:12345-0@lo:400:kworker.0 Request processed in 21us (65us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318983.900870:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2873 00000100:00100000:3.0:1767318983.900887:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985054592:0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.900904:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985054464:0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.900917:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318983.900927:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318983.900930:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985054208:0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.900941:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318983.900952:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985054464 00000100:00100000:3.0:1767318983.900954:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985054464 00000100:00100000:3.0:1767318983.900964:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2874 00000100:00100000:3.0:1767318983.900968:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985054464:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.900985:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985054464:12345-0@lo:400:kworker.0 Request processed in 17us (67us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318983.900989:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2874 00000100:00100000:3.0:1767318983.901001:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985054464:0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.901010:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985054208 00000100:00100000:3.0:1767318983.901012:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985054208 00000100:00100000:3.0:1767318983.901029:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4348 00000100:00100000:3.0:1767318983.901034:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985054208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318983.901065:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985054208:12345-0@lo:400:kworker.0 Request processed in 31us (122us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318983.901071:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4348 00000100:00100000:3.0:1767318983.901092:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985054208:0@lo:400:kworker.0 00000100:00080000:3.0:1767318983.901134:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318983.902046:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985053952:12345-0@lo:400:kworker.0 Request processed in 2096us (2408us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318983.902057:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4347 00000100:00100000:0.0:1767318983.906336:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985053952:0@lo:400:kworker.0 00010000:00010000:2.0:1767318984.539675:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318985.563481:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318986.587625:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318987.483387:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318987.483424:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985054720:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318987.483427:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985054848:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318987.483968:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318987.486441:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985054720 00000100:00100000:3.0:1767318987.486462:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985054720 00000100:00100000:3.0:1767318987.487085:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2028 00000100:00100000:3.0:1767318987.487098:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985054720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318987.487144:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318987.487290:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985054720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 195us (3322us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318987.487298:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2028 00000100:00100000:3.0:1767318987.487386:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985054720:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318987.487922:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318987.488076:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985054848 00000100:00100000:3.0:1767318987.488085:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985054848 00000100:00080000:0.0:1767318987.488087:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318987.488122:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318987.488144:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2029 00000100:00100000:3.0:1767318987.488153:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985054848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318987.488203:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318987.488258:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985054848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318987.488266:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2029 00000100:00100000:1.0:1767318987.488414:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985054848:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318987.611684:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318988.635601:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318988.971968:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318988.972084:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767318988.975063:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920535552 offset 0 length 520: 4 00000100:00100000:1.0:1767318988.977967:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920535424 00000100:00100000:1.0:1767318988.977995:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920535424 00000100:00100000:1.0:1767318988.978126:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920535296 00000100:00100000:1.0:1767318988.978132:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920535296 00000100:00100000:1.0:1767318988.978182:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4349 00000100:00100000:1.0:1767318988.978202:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920535424:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767318988.978480:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318988.978508:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4735 00000100:00100000:3.0:1767318988.978516:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920535296:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767318988.978548:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318988.978657:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920535296:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 140us (6572us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318988.978666:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4735 00000100:00100000:1.0:1767318988.978775:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920535424:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 581us (6826us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318988.978784:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4349 00000100:00100000:0.0:1767318989.019345:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318963/real 1767318963] req@ffff8ce1b5beaa00 x1853163985048448/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318988 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318989.019394:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5beaa00 x1853163985048448/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318988 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767318989.019414:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318989.019439:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318989.019443:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985048448:0@lo:250:kworker.0 00000100:00080000:1.0:1767318989.019569:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318989.019581:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318989.019586:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318989.019594:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4211 00000100:00080000:1.0:1767318989.019600:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318989.019609:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318989.019666:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18d180 x1853163985054976/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318989.019701:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318989.019706:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318989.019708:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318989.019712:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4232 00000100:00080000:1.0:1767318989.019715:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318989.019719:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318989.019737:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18d880 x1853163985055104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318989.019767:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318989.019771:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318989.019773:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318989.019778:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4232 00000100:00080000:1.0:1767318989.019781:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318989.019784:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767318989.019850:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985054976:0@lo:250:kworker.0 00000100:00100000:1.0:1767318989.019901:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18df80 x1853163985055232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:0.0:1767318989.019923:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985054976 offset 0 length 520: 4 00000100:00080000:1.0:1767318989.019960:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318989.019965:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318989.019968:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318989.019974:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4232 00000100:00080000:1.0:1767318989.019977:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318989.019983:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318989.019999:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ce00 x1853163985055488/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318989.020002:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985055360:0@lo:400:kworker.0 00000100:00080000:1.0:1767318989.020070:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318989.020075:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318989.020078:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318989.020084:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4232 00000100:00080000:1.0:1767318989.020089:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318989.020093:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318989.020103:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318989.020115:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c700 x1853163985055744/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318989.020216:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318989.020225:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985056000:0@lo:400:kworker.0 00000100:00100000:3.0:1767318989.020250:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318989.020284:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056000 00000100:00100000:3.0:1767318989.020291:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056000 00000100:00100000:3.0:1767318989.020343:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2875 00000100:00100000:3.0:1767318989.020350:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985056000:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318989.020411:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985056000:12345-0@lo:400:kworker.0 Request processed in 60us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318989.020418:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2875 00000100:00080000:3.0:1767318989.020443:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318989.020455:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985055616:0@lo:400:kworker.0 00000100:00100000:3.0:1767318989.020481:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318989.020520:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318989.020535:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985055872:0@lo:400:kworker.0 00000100:00100000:2.0:1767318989.020556:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985055360 00000100:00100000:2.0:1767318989.020566:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985055360 00000100:00100000:3.0:1767318989.020567:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318989.020612:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985055872 00000100:00100000:3.0:1767318989.020618:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985055872 00000100:00100000:2.0:1767318989.020637:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985055616 00000100:00100000:2.0:1767318989.020644:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985055616 00000100:00100000:3.0:1767318989.020662:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2876 00000100:00100000:3.0:1767318989.020675:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985055872:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318989.020690:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4350 00000100:00100000:2.0:1767318989.020703:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985055360:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318989.020728:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985055872:12345-0@lo:400:kworker.0 Request processed in 55us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318989.020737:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2876 00000100:00100000:3.0:1767318989.020795:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985056000:0@lo:400:kworker.0 00000100:00100000:0.0:1767318989.021079:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985055104:0@lo:38:kworker.0 00000100:00100000:3.0:1767318989.022661:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985055360:0@lo:400:kworker.0 00000100:00100000:3.0:1767318989.022730:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985055872:0@lo:400:kworker.0 00000100:00100000:1.0:1767318989.022972:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4351 00000100:00100000:1.0:1767318989.023000:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985055616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318989.023128:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985055616:12345-0@lo:400:kworker.0 Request processed in 142us (2646us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318989.023139:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4351 00000100:00080000:1.0:1767318989.023244:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318989.023254:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318989.023261:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985055616:0@lo:400:kworker.0 00000100:00100000:2.0:1767318989.023345:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985055360:12345-0@lo:400:kworker.0 Request processed in 1548us (2150us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318989.023355:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4350 00000100:00100000:1.0:1767318989.027627:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318989.027699:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985055232:0@lo:38:kworker.0 00000100:00100000:2.0:1767318989.027735:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985055104 00000100:00100000:1.0:1767318989.027750:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318989.027770:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985055104 00000100:00100000:1.0:1767318989.027877:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985055488:0@lo:38:kworker.0 00000100:00100000:1.0:1767318989.027961:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318989.028069:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985055744:0@lo:38:kworker.0 00000100:00100000:2.0:1767318989.028078:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985055488 00000100:00100000:2.0:1767318989.028084:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985055488 00000100:00100000:1.0:1767318989.028097:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318989.028113:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985055744 00000100:00100000:2.0:1767318989.028117:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985055744 00000100:00100000:2.0:1767318989.028150:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4736 00000100:00100000:2.0:1767318989.028161:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985055104:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318989.028227:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4737 00000100:00100000:1.0:1767318989.028237:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985055488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318989.028243:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767318989.028264:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318989.028309:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985055488:12345-0@lo:38:kworker.0 Request processed in 72us (347us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318989.028317:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4737 00000100:00100000:1.0:1767318989.028341:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4738 00000100:00100000:1.0:1767318989.028346:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985055744:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318989.028355:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318989.028501:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985055104:12345-0@lo:38:kworker.0 Request processed in 339us (879us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318989.028509:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4736 00000100:00100000:1.0:1767318989.028560:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985055744:12345-0@lo:38:kworker.0 Request processed in 209us (462us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318989.028569:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4738 00000100:00080000:1.0:1767318989.028615:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318989.028621:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318989.028625:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985055488:0@lo:38:kworker.0 00000100:00080000:1.0:1767318989.028633:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318989.028636:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318989.028639:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985055744:0@lo:38:kworker.0 00000100:00080000:1.0:1767318989.028649:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318989.028652:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318989.028656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985055104:0@lo:38:kworker.0 00000100:00100000:3.0:1767318989.028826:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985055232 00000100:00100000:3.0:1767318989.028835:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985055232 00000100:00100000:3.0:1767318989.028885:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1674 00000100:00100000:3.0:1767318989.028900:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985055232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318989.028912:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318989.028955:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985055232:12345-0@lo:38:kworker.0 Request processed in 62us (1205us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318989.028961:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1674 00000100:00080000:1.0:1767318989.029210:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318989.029215:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318989.029219:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985055232:0@lo:38:kworker.0 00010000:00010000:2.0:1767318989.659401:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318990.683854:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767318991.709890:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318992.603578:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985056128:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318992.603649:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318992.603698:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056128 00000100:00100000:2.0:1767318992.603706:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056128 00000100:00100000:2.0:1767318992.603776:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2030 00000100:00100000:2.0:1767318992.603787:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985056128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318992.603816:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767318992.603863:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985056128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (213us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318992.603870:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2030 00000100:00100000:2.0:1767318992.603913:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985056128:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318992.604102:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318992.604288:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985056256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318992.604319:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318992.604396:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056256 00000100:00100000:1.0:1767318992.604401:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056256 00000100:00100000:1.0:1767318992.604449:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2031 00000100:00100000:1.0:1767318992.604458:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985056256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318992.604476:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767318992.604517:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985056256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (197us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318992.604523:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2031 00000100:00100000:1.0:1767318992.604554:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985056256:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318992.604696:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318992.731400:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318993.755741:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318994.093047:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767318994.093094:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767318994.093228:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920535680 00000100:00100000:1.0:1767318994.093239:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920535680 00000100:00100000:1.0:1767318994.093291:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920535808 00000100:00100000:1.0:1767318994.093295:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920535808 00000100:00100000:1.0:1767318994.093362:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4352 00000100:00080000:2.0:1767318994.093370:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318994.093372:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920535808:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767318994.093387:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4739 00000100:00100000:2.0:1767318994.093393:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920535680:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767318994.093416:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318994.093454:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920535808:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318994.093462:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4352 00000100:00100000:2.0:1767318994.096309:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920535680:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 2901us (3258us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318994.096324:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4739 00000100:00080000:2.0:1767318994.140055:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318994.140066:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318994.140069:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318994.140073:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318994.140077:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318994.140085:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4237 00000100:00080000:2.0:1767318994.140090:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318994.140098:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318994.140136:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985056384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318994.140187:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318994.140191:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318994.140193:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318994.140197:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4237 00000100:00080000:2.0:1767318994.140199:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318994.140202:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318994.140216:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0a80 x1853163985056512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318994.140264:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318994.140269:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318994.140271:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318994.140275:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4237 00000100:00080000:2.0:1767318994.140278:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318994.140281:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318994.140292:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1f80 x1853163985056768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318994.140317:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318994.140321:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318994.140323:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318994.140327:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4237 00000100:00080000:2.0:1767318994.140329:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318994.140334:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767318994.140343:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985057024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318994.140384:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985056896:0@lo:400:kworker.0 00000100:00100000:2.0:1767318994.140445:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318994.140508:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318994.140515:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985057152:0@lo:400:kworker.0 00000100:00100000:2.0:1767318994.140538:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318994.140571:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057152 00000100:00100000:2.0:1767318994.140578:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057152 00000100:00100000:2.0:1767318994.140632:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2877 00000100:00100000:2.0:1767318994.140640:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985057152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318994.140685:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985057152:12345-0@lo:400:kworker.0 Request processed in 45us (145us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318994.140693:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2877 00000100:00080000:2.0:1767318994.140714:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318994.140728:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056896 00000100:00100000:2.0:1767318994.140732:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056896 00000100:00100000:2.0:1767318994.140761:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4353 00000100:00100000:2.0:1767318994.140767:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985056896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318994.140795:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985056896:12345-0@lo:400:kworker.0 Request processed in 28us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318994.140801:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4353 00000100:00080000:2.0:1767318994.140813:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318994.140836:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985057152:0@lo:400:kworker.0 00000100:00100000:2.0:1767318994.140845:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985056896:0@lo:400:kworker.0 00000100:00100000:1.0:1767318994.140950:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985056384:0@lo:38:kworker.0 00000100:00100000:1.0:1767318994.140987:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318994.141020:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985056640:0@lo:400:kworker.0 00000100:00100000:1.0:1767318994.141025:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985056512:0@lo:38:kworker.0 00000100:00100000:0.0:1767318994.141129:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985057280:0@lo:400:kworker.0 00000100:00100000:2.0:1767318994.141190:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056384 00000100:00100000:2.0:1767318994.141194:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056384 00000100:00100000:2.0:1767318994.141213:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4740 00000100:00100000:2.0:1767318994.141217:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985056384:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318994.141235:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318994.141838:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318994.141905:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056640 00000100:00100000:3.0:1767318994.141912:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056640 00000100:00100000:3.0:1767318994.141991:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4354 00000100:00100000:3.0:1767318994.142002:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985056640:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318994.142057:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985056640:12345-0@lo:400:kworker.0 Request processed in 57us (222us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318994.143404:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4354 00000100:00100000:3.0:1767318994.143512:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985056640:0@lo:400:kworker.0 00000100:00100000:1.0:1767318994.145019:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318994.145093:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985056768:0@lo:38:kworker.0 00000100:00100000:1.0:1767318994.145141:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318994.145190:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985057024:0@lo:38:kworker.0 00000100:00100000:1.0:1767318994.145213:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318994.145267:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056512 00000100:00100000:1.0:1767318994.145274:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056512 00000100:00080000:3.0:1767318994.145301:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318994.145316:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1675 00000100:00100000:1.0:1767318994.145327:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985056512:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318994.145353:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985056768 00000100:00100000:3.0:1767318994.145364:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985056768 00000100:00100000:3.0:1767318994.145403:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057024 00000100:00100000:3.0:1767318994.145406:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057024 00000100:00100000:3.0:1767318994.145420:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4741 00000100:00100000:3.0:1767318994.145428:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985056768:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767318994.145532:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318994.145572:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057280 00000100:00100000:0.0:1767318994.145576:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057280 00000100:00100000:0.0:1767318994.145621:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2878 00000100:00100000:0.0:1767318994.145628:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985057280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318994.145674:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985057280:12345-0@lo:400:kworker.0 Request processed in 47us (143us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318994.145682:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2878 00000100:00100000:0.0:1767318994.145707:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4742 00000100:00100000:0.0:1767318994.145711:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985057024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318994.145745:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318994.145782:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985056512:12345-0@lo:38:kworker.0 Request processed in 457us (771us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318994.145789:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1675 00000100:00080000:1.0:1767318994.145827:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318994.145832:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318994.145836:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985056512:0@lo:38:kworker.0 00010000:00020000:3.0:1767318994.145860:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318994.145952:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985056768:12345-0@lo:38:kworker.0 Request processed in 523us (809us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318994.145960:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4741 00000100:00080000:3.0:1767318994.146002:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318994.146026:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985056384:12345-0@lo:38:kworker.0 Request processed in 4803us (5035us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318994.146036:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4740 00000100:00100000:1.0:1767318994.146115:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985057280:0@lo:400:kworker.0 00000100:00080000:1.0:1767318994.146141:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318994.146145:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318994.146148:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985056384:0@lo:38:kworker.0 00000100:00080000:1.0:1767318994.146157:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318994.146159:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318994.146230:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985056768:0@lo:38:kworker.0 00010000:00020000:0.0:1767318994.146285:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318994.146327:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985057024:12345-0@lo:38:kworker.0 Request processed in 614us (1113us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318994.146333:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4742 00000100:00080000:1.0:1767318994.146459:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318994.146463:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318994.146466:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985057024:0@lo:38:kworker.0 00010000:00010000:1.0:1767318994.779983:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318995.803481:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318996.827471:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318997.723999:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318997.724025:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985057408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318997.724097:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318997.724335:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057408 00000100:00100000:3.0:1767318997.724344:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057408 00000100:00100000:3.0:1767318997.724411:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2032 00000100:00100000:0.0:1767318997.724420:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985057536:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318997.724420:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985057408:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318997.724449:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:3.0:1767318997.724449:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767318997.724489:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057536 00000100:00100000:3.0:1767318997.724492:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985057408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318997.724493:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057536 00000100:00100000:3.0:1767318997.724503:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2032 00000100:00100000:0.0:1767318997.724516:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2033 00000100:00100000:0.0:1767318997.724522:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985057536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318997.724537:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767318997.724547:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985057408:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318997.724565:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985057536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (115us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318997.724571:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2033 00000100:00100000:0.0:1767318997.724595:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985057536:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318997.724656:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318997.724660:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767318997.851445:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318998.875733:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318999.213424:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767318999.213475:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767318999.213545:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920536064 00000100:00100000:2.0:1767318999.213556:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920536064 00000100:00100000:0.0:1767318999.213582:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920535936 00000100:00100000:0.0:1767318999.213589:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920535936 00000100:00100000:0.0:1767318999.213705:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4355 00000100:00100000:0.0:1767318999.213716:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920536064:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767318999.213790:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920536064:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318999.213798:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4355 00000100:00100000:0.0:1767318999.213817:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4743 00000100:00100000:0.0:1767318999.213822:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920535936:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767318999.213872:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318999.213900:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920535936:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 76us (425us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318999.213907:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4743 00000100:00080000:3.0:1767318999.214048:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318999.259737:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318999.259757:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318999.259760:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318999.259765:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318999.260112:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318999.260122:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4242 00000100:00080000:3.0:1767318999.260126:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318999.260133:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318999.260218:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc55500 x1853163985057664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318999.260390:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318999.260395:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318999.260398:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318999.260402:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4242 00000100:00080000:3.0:1767318999.260404:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318999.260408:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318999.260423:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc54000 x1853163985057792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318999.260485:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767318999.260508:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318999.260512:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318999.260514:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318999.260517:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4242 00000100:00080000:3.0:1767318999.260520:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318999.260523:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767318999.260538:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc56a00 x1853163985058048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318999.260563:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318999.260567:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318999.260569:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318999.260572:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4242 00000100:00080000:3.0:1767318999.260574:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318999.260579:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767318999.260582:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985057664:0@lo:38:kworker.0 00000100:00100000:3.0:1767318999.260592:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6a00 x1853163985058304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318999.260684:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318999.260695:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985058560:0@lo:400:kworker.0 00000100:00100000:2.0:1767318999.260720:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985057920:0@lo:400:kworker.0 00000100:00100000:1.0:1767318999.260747:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318999.260930:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985058176:0@lo:400:kworker.0 00000100:00100000:2.0:1767318999.260972:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318999.261029:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057664 00000100:00100000:2.0:1767318999.261036:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057664 00000100:00100000:2.0:1767318999.261065:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057920 00000100:00100000:2.0:1767318999.261067:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057920 00000100:00100000:2.0:1767318999.261100:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4356 00000100:00100000:2.0:1767318999.261107:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985057920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318999.261153:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985057920:12345-0@lo:400:kworker.0 Request processed in 46us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318999.261182:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4356 00000100:00100000:2.0:1767318999.261200:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4744 00000100:00100000:2.0:1767318999.261204:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985057664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318999.261221:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318999.261257:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985057664:12345-0@lo:38:kworker.0 Request processed in 52us (511us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318999.261263:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4744 00000100:00100000:2.0:1767318999.261295:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985057920:0@lo:400:kworker.0 00000100:00080000:2.0:1767318999.261332:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318999.261350:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985058432:0@lo:400:kworker.0 00000100:00100000:2.0:1767318999.261376:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318999.261581:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058432 00000100:00100000:2.0:1767318999.261586:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058432 00000100:00100000:2.0:1767318999.261618:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2879 00000100:00100000:2.0:1767318999.261625:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985058432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318999.261662:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985058432:12345-0@lo:400:kworker.0 Request processed in 37us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318999.261668:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2879 00000100:00100000:2.0:1767318999.261692:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985058432:0@lo:400:kworker.0 00000100:00080000:2.0:1767318999.261705:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318999.263855:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318999.263968:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058560 00000100:00100000:2.0:1767318999.263977:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058560 00000100:00100000:2.0:1767318999.264045:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2880 00000100:00100000:2.0:1767318999.264054:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985058560:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318999.264114:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985058560:12345-0@lo:400:kworker.0 Request processed in 62us (267us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318999.264121:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2880 00000100:00100000:0.0:1767318999.264218:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985058560:0@lo:400:kworker.0 00000100:00080000:0.0:1767318999.264240:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318999.280424:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318999.280932:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058176 00000100:00100000:0.0:1767318999.280942:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058176 00000100:00100000:0.0:1767318999.281017:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4357 00000100:00100000:0.0:1767318999.281027:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985058176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318999.281094:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985058176:12345-0@lo:400:kworker.0 Request processed in 68us (677us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318999.281101:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4357 00000100:00080000:2.0:1767318999.281241:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318999.281291:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985058176:0@lo:400:kworker.0 00000100:00080000:1.0:1767318999.292912:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318999.292926:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318999.292931:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985057664:0@lo:38:kworker.0 00000100:00100000:1.0:1767318999.292945:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985057792:0@lo:38:kworker.0 00000100:00100000:1.0:1767318999.293003:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318999.293042:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985058048:0@lo:38:kworker.0 00000100:00100000:1.0:1767318999.293068:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318999.293111:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985058304:0@lo:38:kworker.0 00000100:00100000:1.0:1767318999.293132:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318999.293327:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058048 00000100:00100000:0.0:1767318999.293335:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058048 00000100:00100000:0.0:1767318999.293379:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058304 00000100:00100000:0.0:1767318999.293382:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058304 00000100:00100000:0.0:1767318999.293402:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4745 00000100:00100000:0.0:1767318999.293409:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985058048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318999.293427:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318999.293473:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985058048:12345-0@lo:38:kworker.0 Request processed in 63us (403us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318999.293480:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4745 00000100:00100000:0.0:1767318999.293497:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4746 00000100:00100000:0.0:1767318999.293501:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985058304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318999.293512:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318999.293535:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985058304:12345-0@lo:38:kworker.0 Request processed in 33us (403us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318999.293541:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4746 00000100:00080000:0.0:1767318999.293568:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318999.293572:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318999.293575:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985058304:0@lo:38:kworker.0 00000100:00080000:0.0:1767318999.293582:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318999.293585:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318999.293587:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985058048:0@lo:38:kworker.0 00000100:00100000:3.0:1767318999.293676:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985057792 00000100:00100000:3.0:1767318999.293681:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985057792 00000100:00100000:3.0:1767318999.293713:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1676 00000100:00100000:3.0:1767318999.293719:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985057792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318999.293729:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318999.293760:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985057792:12345-0@lo:38:kworker.0 Request processed in 40us (757us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318999.293766:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1676 00000100:00080000:0.0:1767318999.293919:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318999.293924:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318999.293927:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985057792:0@lo:38:kworker.0 00010000:00010000:1.0:1767318999.899537:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319000.926367:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319001.947468:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319002.843401:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319002.843418:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985058688:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319002.843476:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319002.843847:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058688 00000100:00100000:2.0:1767319002.843849:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319002.843856:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058688 00000100:00100000:2.0:1767319002.843862:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985058816:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319002.843905:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319002.843957:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058816 00000100:00100000:2.0:1767319002.843961:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058816 00000100:00100000:3.0:1767319002.843983:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2034 00000100:00100000:3.0:1767319002.843994:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985058688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319002.844023:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319002.844067:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985058688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (590us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319002.844075:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2034 00000100:00080000:0.0:1767319002.844120:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319002.844125:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319002.844248:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985058688:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319002.848754:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2035 00000100:00100000:2.0:1767319002.848781:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985058816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319002.848815:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319002.849079:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985058816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 313us (5173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319002.849087:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2035 00000100:00100000:3.0:1767319002.849184:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985058816:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319002.973751:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319003.996145:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319004.332643:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319004.332766:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319004.332900:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920536320 00000100:00100000:1.0:1767319004.332913:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920536320 00000100:00100000:1.0:1767319004.333003:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920536192 00000100:00100000:1.0:1767319004.333007:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920536192 00000100:00100000:1.0:1767319004.333043:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4358 00000100:00100000:1.0:1767319004.333055:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920536320:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319004.333148:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4747 00000100:00100000:1.0:1767319004.333213:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920536320:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (534us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319004.333224:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920536192:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767319004.333226:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4358 00010000:00020000:2.0:1767319004.333260:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319004.333345:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920536192:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 181us (575us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319004.333356:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4747 00000100:00080000:0.0:1767319004.335350:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319004.379585:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319004.379594:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319004.379598:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319004.379602:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319004.379606:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319004.379614:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4247 00000100:00080000:2.0:1767319004.379618:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319004.379624:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319004.379664:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1f80 x1853163985058944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319004.379701:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319004.379705:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319004.379707:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319004.379710:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4247 00000100:00080000:2.0:1767319004.379712:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319004.379715:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319004.379728:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985059072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319004.379776:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319004.379780:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319004.379782:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319004.379786:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4247 00000100:00080000:2.0:1767319004.379789:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319004.379793:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319004.379808:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0a80 x1853163985059328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319004.379841:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985059200:0@lo:400:kworker.0 00000100:00080000:2.0:1767319004.379859:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319004.379864:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319004.379867:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319004.379871:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4247 00000100:00080000:2.0:1767319004.379875:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319004.379879:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319004.379897:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc54000 x1853163985059584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319004.379950:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319004.379969:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985059456:0@lo:400:kworker.0 00000100:00100000:2.0:1767319004.380049:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319004.380100:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059200 00000100:00100000:2.0:1767319004.380108:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059200 00000100:00100000:3.0:1767319004.380231:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319004.380242:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985059840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319004.380284:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319004.380482:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985058944:0@lo:38:kworker.0 00000100:00100000:0.0:1767319004.380526:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319004.380562:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059840 00000100:00100000:3.0:1767319004.380571:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059840 00000100:00100000:3.0:1767319004.380638:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2881 00000100:00100000:3.0:1767319004.380649:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a05c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985059840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319004.380724:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a05c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985059840:12345-0@lo:400:kworker.0 Request processed in 77us (438us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319004.380732:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2881 00000100:00100000:2.0:1767319004.382747:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059456 00000100:00100000:2.0:1767319004.383278:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059456 00000100:00100000:2.0:1767319004.383338:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985058944 00000100:00100000:2.0:1767319004.383342:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985058944 00000100:00100000:2.0:1767319004.383363:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4359 00000100:00100000:2.0:1767319004.383372:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985059200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319004.383447:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985059200:12345-0@lo:400:kworker.0 Request processed in 76us (3499us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319004.383664:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4359 00000100:00100000:2.0:1767319004.383682:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4360 00000100:00100000:2.0:1767319004.383688:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985059456:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319004.383721:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985059456:12345-0@lo:400:kworker.0 Request processed in 33us (3672us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319004.383730:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4360 00000100:00100000:2.0:1767319004.383752:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4748 00000100:00100000:2.0:1767319004.383756:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985058944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319004.383777:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319004.383806:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985058944:12345-0@lo:38:kworker.0 Request processed in 48us (3279us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319004.383812:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4748 00000100:00100000:2.0:1767319004.383897:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985059840:0@lo:400:kworker.0 00000100:00100000:2.0:1767319004.384370:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985059200:0@lo:400:kworker.0 00000100:00080000:2.0:1767319004.384411:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319004.384416:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319004.384895:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985059712:0@lo:400:kworker.0 00000100:00100000:2.0:1767319004.384947:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319004.386856:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319004.386867:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319004.386872:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985058944:0@lo:38:kworker.0 00000100:00100000:0.0:1767319004.386884:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985059072:0@lo:38:kworker.0 00000100:00100000:0.0:1767319004.386935:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319004.386978:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059072 00000100:00100000:0.0:1767319004.386985:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059072 00000100:00100000:0.0:1767319004.387025:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1677 00000100:00100000:0.0:1767319004.387031:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985059072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319004.387053:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319004.387085:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985059072:12345-0@lo:38:kworker.0 Request processed in 52us (149us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319004.387092:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1677 00000100:00100000:0.0:1767319004.387147:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059712 00000100:00100000:0.0:1767319004.387151:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059712 00000100:00100000:0.0:1767319004.387221:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2882 00000100:00100000:0.0:1767319004.387229:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985059712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319004.387273:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985059712:12345-0@lo:400:kworker.0 Request processed in 42us (2322us total) trans 0 rc 0/0 00000100:00080000:1.0:1767319004.387275:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319004.387284:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2882 00000100:00100000:3.0:1767319004.387858:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985059456:0@lo:400:kworker.0 00000100:00080000:0.0:1767319004.389872:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319004.389884:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319004.389889:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985059072:0@lo:38:kworker.0 00000100:00100000:0.0:1767319004.389901:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985059328:0@lo:38:kworker.0 00000100:00100000:0.0:1767319004.389960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319004.390001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985059584:0@lo:38:kworker.0 00000100:00100000:0.0:1767319004.390023:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319004.390251:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059328 00000100:00100000:3.0:1767319004.390267:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059328 00000100:00100000:2.0:1767319004.390313:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985059712:0@lo:400:kworker.0 00000100:00100000:3.0:1767319004.390344:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059584 00000100:00100000:3.0:1767319004.390350:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059584 00000100:00100000:3.0:1767319004.390393:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4749 00000100:00100000:3.0:1767319004.390408:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985059328:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319004.390431:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4750 00010000:00020000:3.0:1767319004.390440:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319004.390445:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985059584:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319004.390522:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985059328:12345-0@lo:38:kworker.0 Request processed in 116us (559us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319004.390531:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4749 00010000:00020000:1.0:1767319004.390576:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319004.390626:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985059584:12345-0@lo:38:kworker.0 Request processed in 184us (602us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319004.390634:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4750 00000100:00080000:0.0:1767319004.390763:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319004.390769:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319004.390772:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985059328:0@lo:38:kworker.0 00000100:00080000:0.0:1767319004.390783:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319004.390786:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319004.390789:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985059584:0@lo:38:kworker.0 00010000:00010000:1.0:1767319005.022822:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319006.044547:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319007.067442:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319007.964308:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985059968:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319007.964379:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319007.964459:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985060096:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319007.964486:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319007.964567:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985059968 00000100:00100000:1.0:1767319007.964576:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985059968 00000100:00100000:1.0:1767319007.964650:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060096 00000100:00100000:1.0:1767319007.964653:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060096 00000100:00100000:3.0:1767319007.964903:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2036 00000100:00100000:3.0:1767319007.964917:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985059968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319007.964946:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767319007.964961:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319007.964966:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319007.965002:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985059968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (620us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319007.965010:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2036 00000100:00100000:3.0:1767319007.965031:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2037 00000100:00100000:3.0:1767319007.965038:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985060096:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319007.965048:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985059968:0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319007.965055:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319007.965090:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985060096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 52us (603us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319007.965098:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2037 00000100:00100000:2.0:1767319007.965109:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985060096:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319008.092013:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319009.116031:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319009.457127:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319009.457189:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319009.457343:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920536448 00000100:00100000:0.0:1767319009.457348:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920536576 00000100:00100000:2.0:1767319009.457354:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920536448 00000100:00100000:0.0:1767319009.457356:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920536576 00000100:00100000:2.0:1767319009.457421:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4361 00000100:00100000:0.0:1767319009.457431:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4751 00000100:00100000:2.0:1767319009.457437:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920536576:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319009.457444:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920536448:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319009.457469:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319009.457522:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920536576:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (354us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319009.457531:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4361 00000100:00080000:1.0:1767319009.457533:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319009.466796:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920536448:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 9342us (9666us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319009.466811:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4751 00000100:00080000:1.0:1767319009.502434:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319009.502444:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319009.502447:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319009.502451:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319009.502455:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319009.502463:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4252 00000100:00080000:1.0:1767319009.502468:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319009.502475:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319009.502512:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ea00 x1853163985060224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319009.502538:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319009.502542:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319009.502543:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319009.502546:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4252 00000100:00080000:1.0:1767319009.502548:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319009.502552:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319009.502565:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18f100 x1853163985060352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319009.502606:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319009.502611:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319009.502613:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319009.502616:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4252 00000100:00080000:1.0:1767319009.502619:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319009.502622:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319009.502636:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2d80 x1853163985060608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319009.502679:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319009.502682:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319009.502684:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319009.502688:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4252 00000100:00080000:1.0:1767319009.502690:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319009.502693:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319009.502702:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1180 x1853163985060864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319009.502750:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985060992:0@lo:400:kworker.0 00000100:00100000:1.0:1767319009.502820:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319009.502876:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319009.502882:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985061120:0@lo:400:kworker.0 00000100:00100000:1.0:1767319009.502903:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319009.502934:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060992 00000100:00100000:1.0:1767319009.502942:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060992 00000100:00100000:1.0:1767319009.502995:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985061120 00000100:00100000:1.0:1767319009.502998:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985061120 00000100:00100000:1.0:1767319009.503020:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2883 00000100:00100000:1.0:1767319009.503028:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985060992:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319009.503076:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985060992:12345-0@lo:400:kworker.0 Request processed in 47us (255us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319009.503083:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2883 00000100:00100000:1.0:1767319009.503100:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2884 00000100:00100000:1.0:1767319009.503106:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985061120:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319009.503131:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985061120:12345-0@lo:400:kworker.0 Request processed in 26us (227us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319009.503136:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2884 00000100:00080000:1.0:1767319009.503159:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319009.503189:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319009.503218:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985061120:0@lo:400:kworker.0 00000100:00100000:1.0:1767319009.503227:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985060992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319009.503531:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985060736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319009.503569:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319009.503607:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319009.503613:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985060480:0@lo:400:kworker.0 00000100:00100000:3.0:1767319009.503634:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319009.503670:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060736 00000100:00100000:3.0:1767319009.503674:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060736 00000100:00100000:3.0:1767319009.503707:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060480 00000100:00100000:3.0:1767319009.503709:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060480 00000100:00100000:3.0:1767319009.503731:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4362 00000100:00100000:3.0:1767319009.503738:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985060736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319009.503776:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985060736:12345-0@lo:400:kworker.0 Request processed in 39us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319009.503782:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4362 00000100:00100000:3.0:1767319009.503795:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4363 00000100:00100000:3.0:1767319009.503801:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985060480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319009.503827:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985060480:12345-0@lo:400:kworker.0 Request processed in 26us (192us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319009.503832:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4363 00000100:00080000:3.0:1767319009.503851:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319009.503873:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985060480:0@lo:400:kworker.0 00000100:00100000:3.0:1767319009.503882:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985060736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319009.505766:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985060224:0@lo:38:kworker.0 00000100:00100000:0.0:1767319009.505832:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319009.505867:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985060352:0@lo:38:kworker.0 00000100:00100000:0.0:1767319009.505890:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319009.505932:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985060608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319009.505959:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319009.505985:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985060864:0@lo:38:kworker.0 00000100:00100000:0.0:1767319009.506005:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319009.506040:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060224 00000100:00100000:0.0:1767319009.506047:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060224 00000100:00100000:0.0:1767319009.506076:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060608 00000100:00100000:0.0:1767319009.506078:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060608 00000100:00100000:0.0:1767319009.506092:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060864 00000100:00100000:0.0:1767319009.506094:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060864 00000100:00100000:0.0:1767319009.506112:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4752 00000100:00100000:0.0:1767319009.506118:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985060224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319009.506139:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319009.506196:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985060224:12345-0@lo:38:kworker.0 Request processed in 76us (362us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319009.506203:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4752 00000100:00100000:0.0:1767319009.506218:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4753 00000100:00100000:0.0:1767319009.506222:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985060608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319009.506230:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319009.506252:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985060608:12345-0@lo:38:kworker.0 Request processed in 28us (292us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319009.506257:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4753 00000100:00100000:0.0:1767319009.506268:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4754 00000100:00100000:0.0:1767319009.506271:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985060864:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319009.506279:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319009.506299:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985060864:12345-0@lo:38:kworker.0 Request processed in 26us (292us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319009.506304:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4754 00000100:00080000:0.0:1767319009.506338:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319009.506343:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319009.506346:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985060224:0@lo:38:kworker.0 00000100:00080000:0.0:1767319009.506354:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319009.506358:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319009.506361:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985060608:0@lo:38:kworker.0 00000100:00080000:0.0:1767319009.506370:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319009.506373:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319009.506391:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985060864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319009.507228:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985060352 00000100:00100000:3.0:1767319009.507236:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985060352 00000100:00100000:3.0:1767319009.507269:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1678 00000100:00100000:3.0:1767319009.507274:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985060352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319009.507287:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319009.507339:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985060352:12345-0@lo:38:kworker.0 Request processed in 63us (1447us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319009.507344:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1678 00000100:00080000:1.0:1767319009.507441:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319009.507447:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319009.507450:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985060352:0@lo:38:kworker.0 00010000:00010000:0.0:1767319010.140817:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319011.163474:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319012.187961:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319013.083946:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319013.083970:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985061248:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319013.084032:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319013.084271:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985061248 00000100:00100000:0.0:1767319013.084279:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985061248 00000100:00100000:0.0:1767319013.084340:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2038 00000100:00100000:0.0:1767319013.084350:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985061248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319013.084378:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319013.084420:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985061248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (387us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319013.084428:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2038 00000100:00080000:1.0:1767319013.084507:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319013.084736:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985061248:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319013.085423:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319013.085434:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985061376:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319013.085476:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319013.085559:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985061376 00000100:00100000:0.0:1767319013.085564:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985061376 00000100:00100000:0.0:1767319013.085698:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2039 00000100:00100000:0.0:1767319013.085704:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985061376:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319013.085719:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319013.085893:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985061376:12345-0@lo:13:osp-pre-0-1.0 Request processed in 188us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319013.085900:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2039 00000100:00080000:1.0:1767319013.085958:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319013.085992:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985061376:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319013.211859:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767319013.552941:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920536704 offset 0 length 520: 4 00000100:00100000:3.0:1767319013.553802:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319013.553840:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319013.554004:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920536960 00000100:00100000:1.0:1767319013.554014:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920536960 00000100:00100000:1.0:1767319013.554085:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920536832 00000100:00100000:1.0:1767319013.554089:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920536832 00000100:00100000:1.0:1767319013.554109:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4364 00000100:00100000:1.0:1767319013.554119:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920536960:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319013.554217:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920536960:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (418us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319013.554225:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4364 00000100:00100000:1.0:1767319013.554244:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4755 00000100:00100000:1.0:1767319013.554248:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920536832:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319013.554268:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319013.554293:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920536832:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (452us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319013.554299:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4755 00000100:00080000:3.0:1767319013.554763:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319014.235720:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319014.619310:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319014.619322:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319014.619325:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319014.619329:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319014.619335:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319014.619345:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4257 00000100:00080000:1.0:1767319014.619350:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319014.619357:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319014.619401:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2a00 x1853163985061504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319014.619419:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319014.619424:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767319014.619425:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318989/real 1767318989] req@ffff8ce1af18d180 x1853163985054976/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319014 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319014.619427:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319014.619434:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4257 00000100:00080000:1.0:1767319014.619440:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319014.619445:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319014.619461:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1af18d180 x1853163985054976/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319014 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319014.619467:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0a80 x1853163985061632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319014.619489:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985061504:0@lo:38:kworker.0 00000100:00080000:1.0:1767319014.619507:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319014.619513:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319014.619515:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319014.619519:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4257 00000100:00080000:1.0:1767319014.619522:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319014.619525:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319014.619538:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985061888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319014.619566:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319014.619572:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767319014.619573:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319014.619574:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319014.619580:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4257 00000100:00080000:1.0:1767319014.619582:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319014.619586:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319014.619598:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0e00 x1853163985062144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319014.619642:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985061632:0@lo:38:kworker.0 00000100:00100000:0.0:1767319014.619671:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319014.619674:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985062272:0@lo:400:kworker.0 00000100:00080000:0.0:1767319014.619705:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319014.619705:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319014.619718:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319014.619723:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985054976:0@lo:250:kworker.0 00000100:00100000:1.0:1767319014.619730:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319014.619736:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985062400:0@lo:400:kworker.0 00000100:00100000:1.0:1767319014.619751:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319014.619759:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985061888:0@lo:38:kworker.0 00000100:00100000:1.0:1767319014.619777:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985062272 00000100:00100000:1.0:1767319014.619784:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985062272 00000100:00100000:0.0:1767319014.619789:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319014.619811:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985062144:0@lo:38:kworker.0 00000100:00100000:1.0:1767319014.619821:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985062400 00000100:00100000:1.0:1767319014.619824:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985062400 00000100:00100000:0.0:1767319014.619834:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319014.619843:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2885 00000100:00100000:1.0:1767319014.619851:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985062272:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319014.619892:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985062272:12345-0@lo:400:kworker.0 Request processed in 43us (187us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319014.619896:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2885 00000100:00100000:1.0:1767319014.619910:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2886 00000100:00100000:1.0:1767319014.619915:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985062400:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319014.619930:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985062400:12345-0@lo:400:kworker.0 Request processed in 16us (180us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319014.619934:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2886 00000100:00100000:0.0:1767319014.619960:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985061632 00000100:00100000:0.0:1767319014.619965:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985061632 00000100:00100000:1.0:1767319014.619969:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985062400:0@lo:400:kworker.0 00000100:00100000:1.0:1767319014.619977:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985062272:0@lo:400:kworker.0 00000100:00100000:0.0:1767319014.619993:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1679 00000100:00100000:0.0:1767319014.619999:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985061632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319014.620018:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319014.620056:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985061632:12345-0@lo:38:kworker.0 Request processed in 56us (385us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319014.620063:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1679 00000100:00100000:0.0:1767319014.620087:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985061504 00000100:00100000:0.0:1767319014.620091:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985061504 00000100:00080000:1.0:1767319014.620112:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319014.620113:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985061888 00000100:00100000:0.0:1767319014.620117:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985061888 00000100:00080000:1.0:1767319014.620117:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319014.620121:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319014.620124:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4237 00000100:00080000:1.0:1767319014.620126:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319014.620129:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319014.620132:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985062144 00000100:00100000:0.0:1767319014.620135:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985062144 00000100:00100000:1.0:1767319014.620139:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3480 x1853163985062528/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319014.620156:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4756 00000100:00100000:0.0:1767319014.620181:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985061504:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319014.620186:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4757 00000100:00100000:1.0:1767319014.620194:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985061888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319014.620200:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767319014.620204:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319014.620239:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985061504:12345-0@lo:38:kworker.0 Request processed in 75us (667us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319014.620240:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985061888:12345-0@lo:38:kworker.0 Request processed in 44us (452us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319014.620254:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4757 00000100:00100000:2.0:1767319014.620308:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985061760:0@lo:400:kworker.0 00000100:00100000:2.0:1767319014.620338:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319014.620353:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319014.620358:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985062016:0@lo:400:kworker.0 00000100:00100000:2.0:1767319014.620378:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319014.620416:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985061760 00000100:00100000:2.0:1767319014.620419:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985061760 00000100:00100000:0.0:1767319014.620471:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4756 00000100:00100000:0.0:1767319014.620484:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985062016 00000100:00100000:0.0:1767319014.620487:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985062016 00000100:00100000:0.0:1767319014.620505:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4365 00000100:00100000:0.0:1767319014.620511:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985062016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319014.620558:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985062016:12345-0@lo:400:kworker.0 Request processed in 47us (178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319014.620564:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4365 00000100:00100000:0.0:1767319014.620580:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4758 00000100:00100000:0.0:1767319014.620583:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985062144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319014.620592:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319014.620617:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985062144:12345-0@lo:38:kworker.0 Request processed in 31us (782us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319014.620623:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4758 00000100:00100000:0.0:1767319014.620663:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985062016:0@lo:400:kworker.0 00000100:00080000:3.0:1767319014.620667:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319014.620700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985062528:0@lo:250:kworker.0 00000400:00000100:3.0:1767319014.620719:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985062528 offset 0 length 520: 4 00000100:00080000:3.0:1767319014.621031:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319014.621044:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319014.621049:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985061888:0@lo:38:kworker.0 00000100:00080000:3.0:1767319014.621065:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319014.621069:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319014.621072:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985062144:0@lo:38:kworker.0 00000100:00080000:3.0:1767319014.621084:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319014.621088:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319014.621091:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985061504:0@lo:38:kworker.0 00000100:00080000:3.0:1767319014.621113:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319014.621118:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319014.621120:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985061632:0@lo:38:kworker.0 00000100:00100000:2.0:1767319014.625556:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4366 00000100:00100000:2.0:1767319014.625583:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985061760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319014.625680:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985061760:12345-0@lo:400:kworker.0 Request processed in 110us (5340us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319014.625686:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4366 00000100:00100000:0.0:1767319014.625828:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985061760:0@lo:400:kworker.0 00010000:00010000:2.0:1767319015.259571:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319016.286138:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319017.313706:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319018.204343:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985062784:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319018.204438:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319018.204554:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985062784 00000100:00100000:2.0:1767319018.204564:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985062784 00000100:00100000:0.0:1767319018.204640:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985062656:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319018.204672:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319018.204713:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985062656 00000100:00100000:0.0:1767319018.204717:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985062656 00000100:00100000:2.0:1767319018.204780:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2040 00000100:00100000:2.0:1767319018.204791:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985062784:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319018.204828:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319018.204990:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2041 00000100:00100000:0.0:1767319018.205001:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985062656:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319018.205025:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319018.205061:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985062656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (387us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319018.205068:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2041 00000100:00100000:0.0:1767319018.205127:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985062656:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319018.205211:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319018.205216:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319018.210438:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985062784:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319018.210534:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985062784:12345-0@lo:13:osp-pre-1-1.0 Request processed in 5743us (6094us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319018.210550:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2040 00010000:00010000:1.0:1767319018.331533:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319018.666203:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319018.666622:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537216 00000100:00100000:2.0:1767319018.666641:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537216 00000100:00100000:2.0:1767319018.666828:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4367 00000100:00100000:2.0:1767319018.666844:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920537216:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319018.666952:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920537216:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 112us (756us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319018.666962:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4367 00000100:00100000:3.0:1767319018.667095:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:0.0:1767319018.671662:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319018.673083:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537088 00000100:00100000:1.0:1767319018.673098:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537088 00000100:00100000:1.0:1767319018.673250:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4759 00000100:00100000:1.0:1767319018.673259:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920537088:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319018.673281:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319018.673360:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920537088:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 101us (6267us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319018.673368:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4759 00010000:00010000:1.0:1767319019.355641:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319019.742840:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319019.742850:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319019.742854:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319019.742858:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319019.742863:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319019.742873:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4262 00000100:00080000:2.0:1767319019.742878:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319019.742886:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319019.742936:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc56680 x1853163985062912/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319019.742977:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319019.742981:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319019.742983:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319019.742987:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4262 00000100:00080000:2.0:1767319019.742989:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319019.742992:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319019.743011:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc57480 x1853163985063040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319019.743062:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319019.743066:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319019.743076:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319019.743080:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4262 00000100:00080000:2.0:1767319019.743082:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319019.743086:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319019.743098:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985063296/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319019.743115:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319019.743118:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319019.743119:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319019.743122:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4262 00000100:00080000:2.0:1767319019.743125:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319019.743128:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319019.743136:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12300 x1853163985063552/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319019.743215:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985063424:0@lo:400:kworker.0 00000100:00100000:2.0:1767319019.743216:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985063680:0@lo:400:kworker.0 00000100:00100000:3.0:1767319019.743304:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985062912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319019.743313:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985063168:0@lo:400:kworker.0 00000100:00100000:2.0:1767319019.746734:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319019.746808:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319019.746821:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985063808:0@lo:400:kworker.0 00000100:00100000:2.0:1767319019.746852:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319019.746903:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063680 00000100:00100000:2.0:1767319019.746912:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063680 00000100:00100000:2.0:1767319019.746972:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063808 00000100:00100000:2.0:1767319019.746976:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063808 00000100:00100000:2.0:1767319019.747010:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2887 00000100:00100000:2.0:1767319019.747020:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985063680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319019.747085:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985063680:12345-0@lo:400:kworker.0 Request processed in 66us (358us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319019.747092:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2887 00000100:00100000:2.0:1767319019.747358:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2888 00000100:00100000:2.0:1767319019.747366:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985063808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319019.747396:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985063808:12345-0@lo:400:kworker.0 Request processed in 31us (543us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319019.747402:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2888 00000100:00100000:2.0:1767319019.747458:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985063808:0@lo:400:kworker.0 00000100:00100000:2.0:1767319019.747471:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985063680:0@lo:400:kworker.0 00000100:00100000:0.0:1767319019.748020:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319019.748140:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063424 00000100:00100000:0.0:1767319019.748147:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063424 00000100:00100000:0.0:1767319019.748211:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4368 00000100:00100000:0.0:1767319019.748220:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985063424:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319019.748259:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985063424:12345-0@lo:400:kworker.0 Request processed in 41us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319019.748266:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4368 00000100:00080000:0.0:1767319019.748288:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319019.748315:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985063424:0@lo:400:kworker.0 00000100:00100000:3.0:1767319019.749369:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319019.749482:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985063040:0@lo:38:kworker.0 00000100:00100000:3.0:1767319019.749515:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319019.749547:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985063296:0@lo:38:kworker.0 00000100:00100000:3.0:1767319019.749569:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319019.749624:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063040 00000100:00100000:2.0:1767319019.749632:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063040 00000100:00100000:2.0:1767319019.749665:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1680 00000100:00100000:2.0:1767319019.749671:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985063040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319019.749693:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319019.749729:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985063040:12345-0@lo:38:kworker.0 Request processed in 58us (214us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319019.749736:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1680 00000100:00100000:3.0:1767319019.752612:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985063552:0@lo:38:kworker.0 00000100:00100000:3.0:1767319019.752661:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319019.753195:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985062912 00000100:00100000:0.0:1767319019.753204:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985062912 00000100:00100000:0.0:1767319019.753252:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063296 00000100:00100000:0.0:1767319019.753256:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063296 00000100:00100000:0.0:1767319019.753277:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4760 00000100:00100000:0.0:1767319019.753284:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985062912:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319019.753306:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319019.753355:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985062912:12345-0@lo:38:kworker.0 Request processed in 70us (3989us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319019.753362:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4760 00000100:00100000:0.0:1767319019.753379:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4761 00000100:00100000:0.0:1767319019.753383:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985063296:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319019.753392:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319019.753415:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985063296:12345-0@lo:38:kworker.0 Request processed in 30us (3844us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319019.753420:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4761 00000100:00100000:2.0:1767319019.760053:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063552 00000100:00100000:2.0:1767319019.760069:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063552 00000100:00100000:2.0:1767319019.760118:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4762 00000100:00100000:2.0:1767319019.760127:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985063552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319019.760148:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319019.762640:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319019.762816:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063168 00000100:00100000:1.0:1767319019.762824:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063168 00000100:00100000:1.0:1767319019.762953:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4369 00000100:00100000:1.0:1767319019.762960:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985063168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319019.763020:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985063168:12345-0@lo:400:kworker.0 Request processed in 61us (389us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319019.763025:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4369 00000100:00100000:1.0:1767319019.763190:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985063168:0@lo:400:kworker.0 00000100:00080000:0.0:1767319019.763250:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319019.764355:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985063552:12345-0@lo:38:kworker.0 Request processed in 4224us (11688us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319019.764366:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4762 00000100:00080000:3.0:1767319019.766931:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319019.766944:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319019.766949:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985062912:0@lo:38:kworker.0 00000100:00080000:3.0:1767319019.766965:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319019.766969:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319019.766972:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985063040:0@lo:38:kworker.0 00000100:00080000:3.0:1767319019.766993:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319019.766997:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319019.767000:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985063296:0@lo:38:kworker.0 00000100:00080000:3.0:1767319019.767012:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319019.767015:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319019.767018:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985063552:0@lo:38:kworker.0 00010000:00010000:2.0:1767319020.380695:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319021.403386:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319022.427659:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319023.323600:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985063936:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319023.323662:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319023.323814:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319023.323821:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985064064:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319023.323842:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319023.323904:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985063936 00000100:00100000:3.0:1767319023.323912:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985063936 00000100:00100000:3.0:1767319023.323967:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064064 00000100:00100000:3.0:1767319023.323970:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064064 00000100:00100000:3.0:1767319023.323986:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2042 00000100:00100000:3.0:1767319023.323993:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985063936:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319023.324020:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319023.324058:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985063936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (395us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319023.324065:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2042 00000100:00100000:3.0:1767319023.324080:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2043 00000100:00100000:3.0:1767319023.324085:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985064064:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319023.324097:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319023.324116:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985064064:12345-0@lo:13:osp-pre-0-1.0 Request processed in 32us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319023.324121:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2043 00000100:00100000:3.0:1767319023.324151:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985064064:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319023.324161:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985063936:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319023.324239:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319023.324243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319023.453468:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319023.786068:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319023.786245:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537472 00000100:00100000:2.0:1767319023.786271:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537472 00000100:00100000:2.0:1767319023.786390:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4370 00000100:00100000:2.0:1767319023.786400:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920537472:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319023.786499:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920537472:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 101us (462us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319023.786508:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4370 00000100:00080000:1.0:1767319023.787519:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319023.788226:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319023.788313:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537344 00000100:00100000:1.0:1767319023.788324:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537344 00000100:00100000:1.0:1767319023.788392:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4763 00000100:00100000:1.0:1767319023.788399:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920537344:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319023.788420:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319023.788500:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920537344:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 99us (287us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319023.788508:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4763 00010000:00010000:1.0:1767319024.479127:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319024.859487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319024.859504:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319024.859510:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319024.859516:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319024.859522:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319024.859535:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4268 00000100:00080000:1.0:1767319024.859542:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319024.859550:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319024.859923:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0a80 x1853163985064192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319024.859972:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319024.859977:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319024.859979:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319024.859984:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4268 00000100:00080000:1.0:1767319024.859987:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319024.859991:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319024.860008:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2a00 x1853163985064320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319024.860069:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319024.860073:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319024.860075:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319024.860079:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4268 00000100:00080000:1.0:1767319024.860082:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319024.860086:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319024.860099:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985064576/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319024.860124:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319024.860128:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319024.860130:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319024.860134:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4268 00000100:00080000:1.0:1767319024.860136:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319024.860139:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319024.860157:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7800 x1853163985064832/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319024.860365:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985064448:0@lo:400:kworker.0 00000100:00100000:2.0:1767319024.860475:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319024.860749:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319024.860760:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985064704:0@lo:400:kworker.0 00000100:00100000:2.0:1767319024.860790:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319024.860827:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985065088:0@lo:400:kworker.0 00000100:00100000:2.0:1767319024.860857:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319024.860879:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319024.860885:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985064960:0@lo:400:kworker.0 00000100:00100000:2.0:1767319024.860909:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319024.860945:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065088 00000100:00100000:2.0:1767319024.860954:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065088 00000100:00100000:2.0:1767319024.861012:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064960 00000100:00100000:2.0:1767319024.861016:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064960 00000100:00100000:2.0:1767319024.861039:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2889 00000100:00100000:2.0:1767319024.861050:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985065088:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319024.861110:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985065088:12345-0@lo:400:kworker.0 Request processed in 61us (251us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319024.861117:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2889 00000100:00100000:2.0:1767319024.861144:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2890 00000100:00100000:2.0:1767319024.861151:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985064960:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319024.861210:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985064960:12345-0@lo:400:kworker.0 Request processed in 59us (300us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319024.861216:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2890 00000100:00100000:1.0:1767319024.861270:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064448 00000100:00100000:1.0:1767319024.861274:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064448 00000100:00100000:1.0:1767319024.861342:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064704 00000100:00100000:1.0:1767319024.861346:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064704 00000100:00100000:1.0:1767319024.861386:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4371 00000100:00100000:1.0:1767319024.861393:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985064448:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319024.861450:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985064448:12345-0@lo:400:kworker.0 Request processed in 57us (976us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319024.861457:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4371 00000100:00100000:1.0:1767319024.861471:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4372 00000100:00100000:1.0:1767319024.861477:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985064704:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319024.861509:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985064704:12345-0@lo:400:kworker.0 Request processed in 32us (718us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319024.861515:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4372 00000100:00080000:1.0:1767319024.861537:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319024.861819:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985064960:0@lo:400:kworker.0 00000100:00100000:2.0:1767319024.861832:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985065088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319024.861904:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985064704:0@lo:400:kworker.0 00000100:00100000:1.0:1767319024.861914:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985064448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319024.862140:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985064192:0@lo:38:kworker.0 00000100:00100000:0.0:1767319024.862229:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319024.862278:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985064320:0@lo:38:kworker.0 00000100:00100000:0.0:1767319024.862320:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319024.862367:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985064576:0@lo:38:kworker.0 00000100:00100000:0.0:1767319024.862394:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319024.862481:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064320 00000100:00100000:1.0:1767319024.862487:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064320 00000100:00100000:1.0:1767319024.862517:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1681 00000100:00100000:1.0:1767319024.862524:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985064320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319024.862554:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319024.862604:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985064320:12345-0@lo:38:kworker.0 Request processed in 79us (283us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319024.862611:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1681 00000100:00100000:0.0:1767319024.862817:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985064832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319024.862845:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319024.865108:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319024.865126:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319024.865132:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985064320:0@lo:38:kworker.0 00000100:00100000:3.0:1767319024.865448:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064192 00000100:00100000:3.0:1767319024.865463:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064192 00000100:00100000:3.0:1767319024.865577:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064576 00000100:00100000:3.0:1767319024.865769:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064576 00000100:00100000:1.0:1767319024.865796:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985064832 00000100:00100000:1.0:1767319024.865800:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985064832 00000100:00100000:2.0:1767319024.865841:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4764 00000100:00100000:2.0:1767319024.865850:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985064192:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319024.865881:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319024.865970:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985064192:12345-0@lo:38:kworker.0 Request processed in 119us (3740us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319024.865978:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4764 00000100:00100000:2.0:1767319024.865997:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4765 00000100:00100000:2.0:1767319024.866001:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985064576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319024.866010:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319024.866032:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985064576:12345-0@lo:38:kworker.0 Request processed in 30us (3638us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319024.866038:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4765 00000100:00100000:2.0:1767319024.866049:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4766 00000100:00100000:2.0:1767319024.866053:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985064832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319024.866061:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319024.866083:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985064832:12345-0@lo:38:kworker.0 Request processed in 29us (3238us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319024.866089:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4766 00000100:00080000:0.0:1767319024.866151:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319024.866157:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319024.866183:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985064192:0@lo:38:kworker.0 00000100:00080000:0.0:1767319024.866195:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319024.866200:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319024.866203:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985064576:0@lo:38:kworker.0 00000100:00080000:0.0:1767319024.866211:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319024.866214:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319024.866217:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985064832:0@lo:38:kworker.0 00010000:00010000:1.0:1767319025.500077:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319026.523412:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319027.548026:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319028.445064:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319028.445096:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985065216:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319028.445209:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319028.445326:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065216 00000100:00100000:3.0:1767319028.445335:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065216 00000100:00100000:3.0:1767319028.445400:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2044 00000100:00100000:3.0:1767319028.445411:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985065216:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319028.445449:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319028.445502:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985065216:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319028.445511:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2044 00000100:00100000:3.0:1767319028.445566:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985065216:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319028.445988:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319028.446002:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985065344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319028.446040:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319028.446129:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319028.446215:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065344 00000100:00100000:0.0:1767319028.446221:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065344 00000100:00100000:0.0:1767319028.446271:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2045 00000100:00100000:0.0:1767319028.446280:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985065344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319028.446309:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319028.446347:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985065344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (305us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319028.446353:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2045 00000100:00100000:2.0:1767319028.446448:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985065344:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319028.446515:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767319028.571551:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319028.906610:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319028.906711:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537728 00000100:00100000:1.0:1767319028.906721:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537728 00000100:00100000:1.0:1767319028.906802:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4373 00000100:00100000:1.0:1767319028.906809:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920537728:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319028.906866:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920537728:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 58us (266us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319028.906871:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4373 00000100:00080000:2.0:1767319028.906936:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319028.907706:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319028.908016:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537600 00000100:00100000:2.0:1767319028.908021:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537600 00000100:00100000:2.0:1767319028.908086:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4767 00000100:00100000:2.0:1767319028.908093:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920537600:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319028.908116:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319028.908194:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920537600:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 100us (491us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319028.908201:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4767 00010000:00010000:1.0:1767319029.595742:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319029.980014:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319029.980023:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319029.980026:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319029.980030:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319029.980033:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319029.980041:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4273 00000100:00080000:2.0:1767319029.980046:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319029.980052:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319029.980101:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10000 x1853163985065472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319029.980127:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319029.980130:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319029.980132:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319029.980135:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4273 00000100:00080000:2.0:1767319029.980137:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319029.980139:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319029.980151:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13480 x1853163985065600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319029.980225:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319029.980229:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319029.980231:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319029.980234:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4273 00000100:00080000:2.0:1767319029.980236:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319029.980239:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319029.980249:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11180 x1853163985065856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319029.980274:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319029.980277:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319029.980279:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319029.980282:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4273 00000100:00080000:2.0:1767319029.980284:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319029.980287:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319029.980295:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985066112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319029.980337:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985066240:0@lo:400:kworker.0 00000100:00100000:2.0:1767319029.980424:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319029.980491:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319029.980497:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985066368:0@lo:400:kworker.0 00000100:00100000:2.0:1767319029.980518:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319029.980558:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985066240 00000100:00100000:2.0:1767319029.980565:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985066240 00000100:00100000:3.0:1767319029.980835:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985065984:0@lo:400:kworker.0 00000100:00100000:3.0:1767319029.980882:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319029.980910:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319029.980916:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985065728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319029.980945:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319029.980947:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985066368 00000100:00100000:2.0:1767319029.980951:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985066368 00000100:00100000:2.0:1767319029.980972:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2891 00000100:00100000:2.0:1767319029.980980:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985066240:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319029.980988:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2892 00000100:00100000:3.0:1767319029.980996:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985066368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319029.981030:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985066368:12345-0@lo:400:kworker.0 Request processed in 34us (510us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319029.981037:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2892 00000100:00100000:0.0:1767319029.981066:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985065472:0@lo:38:kworker.0 00000100:00100000:1.0:1767319029.981129:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065984 00000100:00100000:1.0:1767319029.981133:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065984 00000100:00100000:1.0:1767319029.981190:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065728 00000100:00100000:1.0:1767319029.981194:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065728 00000100:00100000:1.0:1767319029.981217:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4374 00000100:00100000:1.0:1767319029.981237:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985065984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319029.981294:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985065984:12345-0@lo:400:kworker.0 Request processed in 58us (411us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319029.981300:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4374 00000100:00100000:1.0:1767319029.981314:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4375 00000100:00100000:1.0:1767319029.981321:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985065728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319029.981362:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985065728:12345-0@lo:400:kworker.0 Request processed in 41us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319029.981367:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4375 00000100:00080000:1.0:1767319029.981387:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319029.981427:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985065728:0@lo:400:kworker.0 00000100:00100000:1.0:1767319029.981438:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985065984:0@lo:400:kworker.0 00000100:00100000:0.0:1767319029.981557:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319029.981592:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985065600:0@lo:38:kworker.0 00000100:00100000:0.0:1767319029.981616:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319029.981637:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985065856:0@lo:38:kworker.0 00000100:00100000:0.0:1767319029.981656:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319029.981668:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985066112:0@lo:38:kworker.0 00000100:00100000:0.0:1767319029.981684:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319029.981711:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065600 00000100:00100000:0.0:1767319029.981715:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065600 00000100:00100000:0.0:1767319029.981735:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1682 00000100:00100000:0.0:1767319029.981740:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985065600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319029.981759:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319029.981791:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985065600:12345-0@lo:38:kworker.0 Request processed in 50us (174us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319029.981798:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1682 00000100:00080000:0.0:1767319029.981824:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319029.981828:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319029.981831:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985065600:0@lo:38:kworker.0 00000100:00100000:2.0:1767319029.982394:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985066240:12345-0@lo:400:kworker.0 Request processed in 1412us (1974us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319029.982418:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2891 00000100:00100000:1.0:1767319029.983831:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065472 00000100:00100000:1.0:1767319029.983844:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065472 00000100:00100000:1.0:1767319029.983892:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985065856 00000100:00100000:1.0:1767319029.983896:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985065856 00000100:00100000:1.0:1767319029.983918:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985066112 00000100:00100000:1.0:1767319029.983921:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985066112 00000100:00100000:1.0:1767319029.983940:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4768 00000100:00100000:1.0:1767319029.983948:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985065472:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319029.983969:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319029.984042:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985065472:12345-0@lo:38:kworker.0 Request processed in 94us (2484us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319029.984049:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4768 00000100:00100000:1.0:1767319029.984063:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4769 00000100:00100000:1.0:1767319029.984066:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985065856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319029.984075:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319029.984114:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985065856:12345-0@lo:38:kworker.0 Request processed in 45us (2456us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319029.984121:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4769 00000100:00100000:1.0:1767319029.984133:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4770 00000100:00100000:1.0:1767319029.984137:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985066112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319029.984147:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319029.984200:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985066112:12345-0@lo:38:kworker.0 Request processed in 61us (2514us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319029.984218:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4770 00000100:00080000:1.0:1767319029.984281:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319029.984287:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319029.984291:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985065472:0@lo:38:kworker.0 00000100:00080000:1.0:1767319029.984301:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319029.984304:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319029.984307:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985065856:0@lo:38:kworker.0 00000100:00080000:1.0:1767319029.984315:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319029.984318:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319029.984321:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985066112:0@lo:38:kworker.0 00000100:00100000:3.0:1767319029.987278:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985066368:0@lo:400:kworker.0 00000100:00100000:3.0:1767319029.987311:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985066240:0@lo:400:kworker.0 00010000:00010000:1.0:1767319030.619484:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319031.644702:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319032.671216:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319033.563941:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319033.563966:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985066496:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319033.564033:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319033.564940:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985066496 00000100:00100000:3.0:1767319033.564949:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985066496 00000100:00100000:3.0:1767319033.565013:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2046 00000100:00100000:3.0:1767319033.565023:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985066496:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319033.565059:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319033.565118:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985066496:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (1084us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319033.565126:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2046 00000100:00100000:0.0:1767319033.566039:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985066496:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319033.567366:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985066624:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319033.567431:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319033.567540:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985066624 00000100:00100000:0.0:1767319033.567548:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985066624 00000100:00100000:0.0:1767319033.567611:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2047 00000100:00100000:0.0:1767319033.567620:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985066624:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319033.567648:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319033.567688:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985066624:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (257us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319033.567696:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2047 00000100:00080000:0.0:1767319033.571524:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319033.571535:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319033.576006:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985066624:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319033.691709:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319034.026148:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319034.026237:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537984 00000100:00100000:2.0:1767319034.026247:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537984 00000100:00100000:3.0:1767319034.026258:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:3.0:1767319034.026307:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319034.026314:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920537856 00000100:00100000:2.0:1767319034.026319:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920537856 00000100:00100000:1.0:1767319034.029366:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4376 00000100:00100000:1.0:1767319034.029392:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920537984:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319034.031617:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920537984:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 2233us (5473us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319034.031629:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4376 00000100:00100000:1.0:1767319034.031713:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4771 00000100:00100000:1.0:1767319034.031721:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920537856:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319034.031742:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319034.031769:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920537856:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (5512us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319034.031776:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4771 00010000:00010000:0.0:1767319034.715900:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319035.101602:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319035.101613:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319035.101616:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319035.101620:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319035.101624:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319035.101632:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4278 00000100:00080000:2.0:1767319035.101636:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319035.101643:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319035.101681:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ad9500 x1853163985066752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319035.101715:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319035.101719:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319035.101720:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319035.101724:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4278 00000100:00080000:2.0:1767319035.101726:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319035.101729:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319035.101744:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adb800 x1853163985066880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319035.101769:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985066752:0@lo:38:kworker.0 00000100:00080000:2.0:1767319035.101792:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319035.101797:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319035.101798:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319035.101803:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4278 00000100:00080000:2.0:1767319035.101806:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319035.101809:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319035.101824:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ad9f80 x1853163985067136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319035.101841:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319035.101845:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319035.101849:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319035.101851:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319035.101855:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4278 00000100:00080000:2.0:1767319035.101858:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319035.101862:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319035.101874:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ad9880 x1853163985067392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319035.101925:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985067520:0@lo:400:kworker.0 00000100:00100000:3.0:1767319035.101932:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985067008:0@lo:400:kworker.0 00000100:00100000:2.0:1767319035.101964:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319035.101975:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319035.102025:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985066880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319035.102108:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319035.102138:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985067136:0@lo:38:kworker.0 00000100:00100000:0.0:1767319035.102190:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319035.102208:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985067392:0@lo:38:kworker.0 00000100:00100000:0.0:1767319035.102228:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319035.102256:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985066752 00000100:00100000:0.0:1767319035.102263:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985066752 00000100:00100000:1.0:1767319035.102278:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985067264:0@lo:400:kworker.0 00000100:00100000:0.0:1767319035.102296:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067008 00000100:00100000:0.0:1767319035.102300:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067008 00000100:00100000:1.0:1767319035.102313:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319035.102336:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067136 00000100:00100000:0.0:1767319035.102340:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067136 00000100:00100000:0.0:1767319035.102350:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067392 00000100:00100000:0.0:1767319035.102353:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067392 00000100:00100000:1.0:1767319035.102356:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067264 00000100:00100000:1.0:1767319035.102361:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067264 00000100:00100000:0.0:1767319035.102383:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4377 00000100:00100000:0.0:1767319035.102393:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985067008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319035.102411:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4378 00000100:00100000:1.0:1767319035.102422:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985067264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319035.102442:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985067008:12345-0@lo:400:kworker.0 Request processed in 50us (466us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319035.102451:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4377 00000100:00100000:0.0:1767319035.102473:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4772 00000100:00100000:0.0:1767319035.102477:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985066752:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319035.102483:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985067264:12345-0@lo:400:kworker.0 Request processed in 61us (168us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319035.102490:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4378 00010000:00020000:0.0:1767319035.102501:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319035.102509:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4773 00000100:00100000:1.0:1767319035.102514:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985067136:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319035.102525:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319035.102540:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985066752:12345-0@lo:38:kworker.0 Request processed in 59us (699us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319035.102559:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985067136:12345-0@lo:38:kworker.0 Request processed in 41us (396us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319035.102562:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4772 00000100:00100000:1.0:1767319035.102573:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4773 00000100:00100000:0.0:1767319035.102582:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4774 00000100:00100000:0.0:1767319035.102588:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985067392:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319035.102598:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985066880 00010000:00020000:0.0:1767319035.102599:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319035.102604:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985066880 00000100:00100000:0.0:1767319035.102630:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985067392:12345-0@lo:38:kworker.0 Request processed in 40us (400us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319035.102632:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1683 00000100:00100000:1.0:1767319035.102639:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190a40c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985066880:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319035.102644:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4774 00010000:00020000:1.0:1767319035.102655:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319035.102683:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190a40c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985066880:12345-0@lo:38:kworker.0 Request processed in 43us (584us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767319035.102686:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319035.102688:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1683 00000100:00080000:0.0:1767319035.102694:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319035.102700:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985067136:0@lo:38:kworker.0 00000100:00080000:0.0:1767319035.102711:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319035.102715:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319035.102719:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985067392:0@lo:38:kworker.0 00000100:00080000:0.0:1767319035.102731:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319035.102736:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319035.102739:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985066752:0@lo:38:kworker.0 00000100:00080000:0.0:1767319035.102752:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319035.102755:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319035.102759:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985066880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319035.102852:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985067264:0@lo:400:kworker.0 00000100:00100000:0.0:1767319035.102902:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067520 00000100:00100000:0.0:1767319035.102907:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067520 00000100:00100000:1.0:1767319035.102917:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985067648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319035.102950:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2893 00000100:00100000:0.0:1767319035.102960:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ada300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985067520:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319035.102974:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319035.102999:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067648 00000100:00100000:1.0:1767319035.103002:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067648 00000100:00100000:0.0:1767319035.103004:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ada300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985067520:12345-0@lo:400:kworker.0 Request processed in 46us (1039us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319035.103012:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2893 00000100:00100000:0.0:1767319035.103032:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2894 00000100:00100000:0.0:1767319035.103040:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985067648:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319035.103067:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319035.103073:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319035.103076:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319035.103080:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985067648:12345-0@lo:400:kworker.0 Request processed in 41us (105us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319035.103087:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2894 00000100:00100000:1.0:1767319035.103107:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985067648:0@lo:400:kworker.0 00000100:00100000:2.0:1767319035.104350:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985067520:0@lo:400:kworker.0 00000100:00100000:3.0:1767319035.104631:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985067008:0@lo:400:kworker.0 00010000:00010000:1.0:1767319035.743005:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319036.764298:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319037.799161:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319038.685750:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985067776:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319038.685822:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319038.685860:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067776 00000100:00100000:0.0:1767319038.685869:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067776 00000100:00100000:2.0:1767319038.685898:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319038.685907:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985067904:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319038.685938:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2048 00000100:00100000:2.0:1767319038.685944:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319038.685952:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985067776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319038.685987:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319038.685994:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985067904 00000100:00100000:2.0:1767319038.685999:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985067904 00000100:00100000:2.0:1767319038.686032:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2049 00000100:00100000:0.0:1767319038.686041:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985067776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (218us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319038.686043:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985067904:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319038.686058:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2048 00002000:00000024:2.0:1767319038.686065:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319038.686125:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985067776:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319038.686141:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319038.686147:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319038.695775:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985067904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 9725us (9823us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319038.695791:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2049 00000100:00100000:0.0:1767319038.695952:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985067904:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319038.811474:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319039.145871:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920538112 offset 0 length 520: 4 00000100:00100000:3.0:1767319039.145923:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319039.146032:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319039.148204:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920538368 00000100:00100000:2.0:1767319039.148217:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920538368 00000100:00100000:2.0:1767319039.148341:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920538240 00000100:00100000:2.0:1767319039.148345:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920538240 00000100:00100000:2.0:1767319039.148386:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4379 00000100:00100000:2.0:1767319039.148409:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920538368:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319039.148813:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4775 00000100:00100000:1.0:1767319039.148821:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920538240:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319039.148844:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319039.148915:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920538240:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 93us (2883us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319039.148923:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4775 00000100:00100000:2.0:1767319039.151104:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920538368:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 2694us (5172us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319039.151116:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4379 00000100:00080000:3.0:1767319039.153966:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319039.195564:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319014/real 1767319014] req@ffff8ce1b34d3480 x1853163985062528/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319039 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319039.195601:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b34d3480 x1853163985062528/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319039 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319039.195618:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319039.195639:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319039.195642:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985062528:0@lo:250:kworker.0 00000100:00080000:0.0:1767319039.195711:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319039.195717:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319039.195720:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319039.195723:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4262 00000100:00080000:0.0:1767319039.195726:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319039.195731:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319039.195760:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3480 x1853163985068032/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319039.195783:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319039.195786:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319039.195787:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319039.195790:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4283 00000100:00080000:0.0:1767319039.195791:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319039.195794:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319039.195803:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0e00 x1853163985068160/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319039.195808:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319039.195810:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319039.195812:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319039.195813:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4283 00000100:00080000:0.0:1767319039.195815:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319039.195816:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319039.195822:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2300 x1853163985068288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319039.195858:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319039.195861:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319039.195862:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319039.195863:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4283 00000100:00080000:0.0:1767319039.195865:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319039.195867:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319039.195875:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0a80 x1853163985068544/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319039.195885:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319039.195887:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319039.195888:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319039.195890:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4283 00000100:00080000:0.0:1767319039.195891:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319039.195893:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319039.195898:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2680 x1853163985068800/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319039.195931:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985068928:0@lo:400:kworker.0 00000100:00100000:0.0:1767319039.195982:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319039.196025:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319039.196029:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985069056:0@lo:400:kworker.0 00000100:00100000:0.0:1767319039.196041:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319039.196059:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985068928 00000100:00100000:0.0:1767319039.196066:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985068928 00000100:00100000:3.0:1767319039.196109:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985068672:0@lo:400:kworker.0 00000100:00100000:0.0:1767319039.196112:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069056 00000100:00100000:0.0:1767319039.196118:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069056 00000100:00100000:0.0:1767319039.196146:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2895 00000100:00100000:3.0:1767319039.196154:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319039.196159:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985068928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319039.196232:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985068928:12345-0@lo:400:kworker.0 Request processed in 78us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319039.196240:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2895 00000100:00100000:0.0:1767319039.196257:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2896 00000100:00100000:0.0:1767319039.196265:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985069056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319039.196286:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319039.196296:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985068416:0@lo:400:kworker.0 00000100:00100000:0.0:1767319039.196303:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985069056:12345-0@lo:400:kworker.0 Request processed in 36us (256us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319039.196311:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2896 00000100:00100000:3.0:1767319039.196327:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319039.196352:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985069056:0@lo:400:kworker.0 00000100:00100000:0.0:1767319039.196368:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985068928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319039.196563:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985068672 00000100:00100000:3.0:1767319039.196568:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985068672 00000100:00100000:3.0:1767319039.196607:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985068416 00000100:00100000:3.0:1767319039.196611:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985068416 00000100:00100000:3.0:1767319039.196629:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4380 00000100:00100000:3.0:1767319039.196636:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985068672:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319039.196676:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985068672:12345-0@lo:400:kworker.0 Request processed in 40us (523us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319039.196684:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4380 00000100:00100000:3.0:1767319039.196702:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4381 00000100:00100000:3.0:1767319039.196710:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985068416:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319039.196719:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985068032:0@lo:250:kworker.0 00000100:00100000:3.0:1767319039.196745:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985068416:12345-0@lo:400:kworker.0 Request processed in 35us (417us total) trans 0 rc 0/0 00000400:00000100:1.0:1767319039.196746:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985068032 offset 0 length 520: 4 00000100:00100000:3.0:1767319039.196757:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4381 00000100:00080000:3.0:1767319039.196777:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319039.196800:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985068416:0@lo:400:kworker.0 00000100:00100000:3.0:1767319039.196809:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985068672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319039.202155:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985068160:0@lo:38:kworker.0 00000100:00100000:1.0:1767319039.202251:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319039.202281:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985068288:0@lo:38:kworker.0 00000100:00100000:1.0:1767319039.202306:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319039.202336:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985068544:0@lo:38:kworker.0 00000100:00100000:1.0:1767319039.202356:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319039.202382:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985068800:0@lo:38:kworker.0 00000100:00100000:1.0:1767319039.202828:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319039.202983:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985068160 00000100:00100000:3.0:1767319039.202992:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985068160 00000100:00100000:2.0:1767319039.203004:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985068288 00000100:00100000:2.0:1767319039.203010:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985068288 00000100:00100000:3.0:1767319039.203038:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985068544 00000100:00100000:3.0:1767319039.203043:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985068544 00000100:00100000:2.0:1767319039.203044:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1684 00000100:00100000:2.0:1767319039.203052:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985068288:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319039.203070:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985068800 00010000:00020000:2.0:1767319039.203073:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319039.203075:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985068800 00000100:00100000:3.0:1767319039.203109:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4776 00000100:00100000:3.0:1767319039.203116:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985068160:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319039.203128:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319039.203137:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985068288:12345-0@lo:38:kworker.0 Request processed in 81us (827us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319039.203143:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1684 00000100:00100000:2.0:1767319039.203218:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4777 00000100:00100000:2.0:1767319039.203223:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985068544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319039.203232:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319039.203264:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985068544:12345-0@lo:38:kworker.0 Request processed in 39us (906us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319039.203270:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4777 00000100:00100000:2.0:1767319039.203287:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4778 00000100:00100000:2.0:1767319039.203292:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985068800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319039.203302:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767319039.203323:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767319039.203330:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985068800:12345-0@lo:38:kworker.0 Request processed in 36us (500us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767319039.203332:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319039.203340:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985068160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319039.203342:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4778 00000100:00080000:1.0:1767319039.203359:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319039.203365:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319039.203381:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985068288:0@lo:38:kworker.0 00000100:00080000:1.0:1767319039.203393:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319039.203396:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319039.203399:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985068544:0@lo:38:kworker.0 00000100:00080000:1.0:1767319039.203407:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319039.203410:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319039.203413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985068800:0@lo:38:kworker.0 00000100:00100000:3.0:1767319039.203468:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985068160:12345-0@lo:38:kworker.0 Request processed in 72us (937us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319039.203475:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4776 00010000:00010000:2.0:1767319039.835423:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319040.859446:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319041.883550:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319042.907363:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319043.803966:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319043.803989:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985069184:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319043.804068:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319043.804198:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069184 00000100:00100000:2.0:1767319043.804207:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069184 00000100:00100000:2.0:1767319043.804340:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2050 00000100:00100000:2.0:1767319043.804351:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985069184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319043.804389:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319043.804437:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985069184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (369us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319043.804445:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2050 00000100:00100000:2.0:1767319043.804488:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985069184:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319043.804858:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319043.804869:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985069312:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319043.804899:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319043.804986:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069312 00000100:00100000:1.0:1767319043.804991:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069312 00000100:00080000:3.0:1767319043.805069:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319043.805074:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319043.805458:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2051 00000100:00100000:1.0:1767319043.805468:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985069312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319043.805486:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319043.805569:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985069312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (669us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319043.805576:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2051 00000100:00100000:0.0:1767319043.805683:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985069312:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319043.931608:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319044.267923:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319044.268088:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319044.276730:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920538496 00000100:00100000:0.0:1767319044.276745:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920538496 00000100:00100000:0.0:1767319044.277057:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920538624 00000100:00100000:0.0:1767319044.277062:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920538624 00000100:00080000:3.0:1767319044.278886:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319044.281182:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4382 00000100:00100000:2.0:1767319044.281216:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920538624:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319044.281336:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920538624:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 139us (13248us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319044.281345:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4382 00000100:00100000:2.0:1767319044.281375:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4779 00000100:00100000:2.0:1767319044.281380:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920538496:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319044.281417:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319044.281446:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920538496:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 64us (13533us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319044.281452:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4779 00000100:00080000:1.0:1767319044.317190:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319044.317200:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319044.317203:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319044.317211:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319044.317216:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319044.317223:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4287 00000100:00080000:1.0:1767319044.317228:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319044.317235:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319044.317273:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6d80 x1853163985069440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319044.317295:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319044.317299:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319044.317300:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319044.317303:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4287 00000100:00080000:1.0:1767319044.317305:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319044.317311:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319044.317321:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5c00 x1853163985069568/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319044.317360:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319044.317364:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319044.317365:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319044.317368:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4287 00000100:00080000:1.0:1767319044.317370:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319044.317374:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319044.317387:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6a00 x1853163985069824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319044.317400:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1767319044.317405:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319044.317410:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319044.317412:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319044.317416:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4287 00000100:00080000:1.0:1767319044.317418:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319044.317421:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319044.317426:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985069952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319044.317435:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7b80 x1853163985070080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319044.317511:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319044.317573:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069952 00000100:00100000:3.0:1767319044.317576:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985069696:0@lo:400:kworker.0 00000100:00100000:1.0:1767319044.317581:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069952 00000100:00100000:3.0:1767319044.317612:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319044.317647:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069696 00000100:00100000:3.0:1767319044.317651:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069696 00000100:00100000:1.0:1767319044.317703:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4383 00000100:00100000:1.0:1767319044.317713:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985069952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319044.317759:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985069952:12345-0@lo:400:kworker.0 Request processed in 49us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319044.317764:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4383 00000100:00100000:2.0:1767319044.317814:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985069440:0@lo:38:kworker.0 00000100:00100000:2.0:1767319044.317835:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319044.317851:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985069568:0@lo:38:kworker.0 00000100:00100000:2.0:1767319044.317866:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319044.317877:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985069824:0@lo:38:kworker.0 00000100:00100000:2.0:1767319044.317887:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319044.317895:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985070080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319044.317905:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319044.317922:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069440 00000100:00100000:2.0:1767319044.317925:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069440 00000100:00100000:2.0:1767319044.317935:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069824 00000100:00100000:2.0:1767319044.317936:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069824 00000100:00100000:2.0:1767319044.317940:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070080 00000100:00100000:2.0:1767319044.317941:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070080 00000100:00100000:2.0:1767319044.317947:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4780 00000100:00100000:2.0:1767319044.317950:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985069440:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319044.317965:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319044.318007:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985069440:12345-0@lo:38:kworker.0 Request processed in 55us (170us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319044.318013:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4780 00000100:00100000:2.0:1767319044.318026:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4781 00000100:00100000:2.0:1767319044.318030:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985069824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319044.318039:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319044.318060:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985069824:12345-0@lo:38:kworker.0 Request processed in 29us (171us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319044.318064:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4781 00000100:00100000:2.0:1767319044.318072:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4782 00000100:00100000:2.0:1767319044.318074:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985070080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319044.318082:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319044.318104:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985070080:12345-0@lo:38:kworker.0 Request processed in 27us (197us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319044.318109:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4782 00000100:00100000:2.0:1767319044.318127:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985069568 00000100:00100000:2.0:1767319044.318130:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985069568 00000100:00100000:2.0:1767319044.318147:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1685 00000100:00100000:2.0:1767319044.318152:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985069568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319044.318161:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319044.318205:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985069568:12345-0@lo:38:kworker.0 Request processed in 52us (337us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319044.318211:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1685 00000100:00080000:2.0:1767319044.318240:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319044.318244:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319044.318248:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985069440:0@lo:38:kworker.0 00000100:00080000:2.0:1767319044.318257:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319044.318259:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319044.318260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985069568:0@lo:38:kworker.0 00000100:00080000:2.0:1767319044.318264:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319044.318267:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319044.318268:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985069824:0@lo:38:kworker.0 00000100:00080000:2.0:1767319044.318272:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319044.318274:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319044.318276:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985070080:0@lo:38:kworker.0 00000100:00080000:2.0:1767319044.318321:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319044.318339:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985069952:0@lo:400:kworker.0 00000100:00100000:3.0:1767319044.322478:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4384 00000100:00100000:3.0:1767319044.322512:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985069696:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319044.322623:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985069696:12345-0@lo:400:kworker.0 Request processed in 132us (5020us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319044.322631:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4384 00000100:00100000:0.0:1767319044.322656:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985069696:0@lo:400:kworker.0 00010000:00010000:2.0:1767319044.955566:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319045.979420:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319047.003463:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319048.027420:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319048.923749:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985070208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319048.923820:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319048.923975:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070208 00000100:00100000:1.0:1767319048.923984:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070208 00000100:00100000:1.0:1767319048.924050:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2052 00000100:00100000:1.0:1767319048.924061:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985070208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319048.924100:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319048.924147:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985070208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 89us (327us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319048.924155:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2052 00000100:00100000:1.0:1767319048.924222:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985070208:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319048.924564:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319048.926779:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985070336:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319048.926847:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319048.927213:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070336 00000100:00100000:0.0:1767319048.927223:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070336 00000100:00100000:0.0:1767319048.927294:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2053 00000100:00100000:0.0:1767319048.927305:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985070336:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319048.927335:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319048.927390:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985070336:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (541us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319048.927398:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2053 00000100:00080000:3.0:1767319048.929524:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319048.929640:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985070336:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319049.053007:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319049.386475:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319049.386518:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319049.386760:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920538880 00000100:00100000:2.0:1767319049.386770:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920538880 00000100:00100000:2.0:1767319049.386999:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920538752 00000100:00100000:2.0:1767319049.387004:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920538752 00000100:00100000:2.0:1767319049.387029:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4385 00000100:00100000:2.0:1767319049.387042:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920538880:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319049.387123:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920538880:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (670us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319049.387132:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4385 00000100:00100000:2.0:1767319049.387152:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4783 00000100:00100000:2.0:1767319049.387156:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920538752:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319049.387222:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319049.387250:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920538752:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (731us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319049.387257:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4783 00000100:00080000:0.0:1767319049.388216:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319049.436008:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319049.436021:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319049.436024:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319049.436028:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319049.436032:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319049.436040:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4292 00000100:00080000:1.0:1767319049.436045:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319049.436052:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319049.436090:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7b80 x1853163985070464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319049.436121:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319049.436125:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319049.436127:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319049.436130:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4292 00000100:00080000:1.0:1767319049.436133:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319049.436136:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319049.436148:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6a00 x1853163985070592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319049.436214:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319049.436218:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319049.436220:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319049.436223:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4292 00000100:00080000:1.0:1767319049.436226:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319049.436230:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319049.436241:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6d80 x1853163985070848/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319049.436253:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1767319049.436257:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319049.436260:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319049.436261:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319049.436264:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4292 00000100:00080000:1.0:1767319049.436266:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319049.436268:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985070976:0@lo:400:kworker.0 00000100:00080000:1.0:1767319049.436269:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319049.436281:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7800 x1853163985071104/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319049.436333:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319049.436400:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985070720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.436422:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319049.436453:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985070464:0@lo:38:kworker.0 00000100:00100000:2.0:1767319049.436481:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319049.436506:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985070592:0@lo:38:kworker.0 00000100:00100000:2.0:1767319049.436529:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319049.436565:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985070848:0@lo:38:kworker.0 00000100:00100000:2.0:1767319049.436585:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319049.436600:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985071104:0@lo:38:kworker.0 00000100:00100000:2.0:1767319049.436618:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319049.436631:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070592 00000100:00100000:0.0:1767319049.436640:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070592 00000100:00100000:2.0:1767319049.436644:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070976 00000100:00100000:2.0:1767319049.436647:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070976 00000100:00100000:0.0:1767319049.436675:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1686 00000100:00100000:0.0:1767319049.436683:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985070592:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319049.436684:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070720 00000100:00100000:2.0:1767319049.436686:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070720 00000100:00100000:2.0:1767319049.436698:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070464 00000100:00100000:2.0:1767319049.436699:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070464 00010000:00020000:0.0:1767319049.436705:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319049.436705:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985070848 00000100:00100000:2.0:1767319049.436707:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985070848 00000100:00100000:2.0:1767319049.436713:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985071104 00000100:00100000:2.0:1767319049.436715:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985071104 00000100:00100000:2.0:1767319049.436723:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4386 00000100:00100000:2.0:1767319049.436729:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985070976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319049.436745:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985070592:12345-0@lo:38:kworker.0 Request processed in 62us (215us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319049.436752:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1686 00000100:00100000:2.0:1767319049.436764:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985070976:12345-0@lo:400:kworker.0 Request processed in 35us (431us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319049.436770:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4386 00000100:00080000:0.0:1767319049.436772:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319049.436781:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4387 00000100:00100000:2.0:1767319049.436787:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985070720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319049.436788:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4784 00000100:00100000:0.0:1767319049.436792:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985070464:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319049.436801:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319049.436810:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985070720:12345-0@lo:400:kworker.0 Request processed in 24us (387us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319049.436816:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4387 00000100:00100000:0.0:1767319049.436819:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985070464:12345-0@lo:38:kworker.0 Request processed in 26us (338us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319049.436824:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4784 00000100:00100000:2.0:1767319049.436827:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4785 00000100:00100000:2.0:1767319049.436995:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ada680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985070848:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319049.437006:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319049.437012:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4786 00000100:00100000:0.0:1767319049.437016:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985071104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319049.437026:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319049.437030:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ada680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985070848:12345-0@lo:38:kworker.0 Request processed in 35us (444us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319049.437036:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4785 00000100:00100000:0.0:1767319049.437045:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985071104:12345-0@lo:38:kworker.0 Request processed in 28us (426us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319049.437050:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4786 00000100:00100000:2.0:1767319049.437076:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985070976:0@lo:400:kworker.0 00000100:00100000:2.0:1767319049.437086:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985070720:0@lo:400:kworker.0 00000100:00080000:2.0:1767319049.437112:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319049.437117:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319049.437120:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985070464:0@lo:38:kworker.0 00000100:00080000:2.0:1767319049.437128:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319049.437131:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319049.437133:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985070592:0@lo:38:kworker.0 00000100:00080000:2.0:1767319049.437143:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319049.437146:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319049.437148:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985070848:0@lo:38:kworker.0 00000100:00080000:2.0:1767319049.437155:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319049.437157:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319049.437160:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985071104:0@lo:38:kworker.0 00010000:00010000:1.0:1767319050.075915:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319051.099413:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319052.123378:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319053.147408:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319054.043889:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319054.043913:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985071232:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319054.043992:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319054.044436:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985071232 00000100:00100000:0.0:1767319054.044447:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985071232 00000100:00100000:0.0:1767319054.044530:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2054 00000100:00100000:0.0:1767319054.044540:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985071232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319054.044571:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319054.044620:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985071232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (627us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319054.044627:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2054 00000100:00080000:2.0:1767319054.044921:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319054.045045:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985071232:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319054.047293:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985071360:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319054.047360:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319054.053651:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985071360 00000100:00100000:3.0:1767319054.053667:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985071360 00000100:00100000:3.0:1767319054.053770:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2055 00000100:00100000:3.0:1767319054.053782:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985071360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319054.053824:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319054.053917:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985071360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 138us (6556us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319054.053925:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2055 00000100:00080000:2.0:1767319054.054243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319054.054275:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985071360:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319054.171505:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319054.505956:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319054.506039:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319054.515519:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920539008 00000100:00100000:0.0:1767319054.515547:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920539008 00000100:00100000:0.0:1767319054.515592:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920539136 00000100:00100000:0.0:1767319054.515596:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920539136 00000100:00100000:0.0:1767319054.515661:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4388 00000100:00100000:0.0:1767319054.515673:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920539136:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319054.515752:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920539136:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (9713us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319054.515761:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4388 00000100:00100000:0.0:1767319054.515782:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4787 00000100:00100000:0.0:1767319054.515787:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920539008:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319054.515836:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319054.515865:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920539008:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 77us (9920us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319054.515873:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4787 00000100:00080000:3.0:1767319054.516370:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319054.556637:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319054.556649:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319054.556653:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319054.556657:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319054.556662:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319054.556672:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4297 00000100:00080000:2.0:1767319054.556678:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319054.556686:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319054.556730:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ad9880 x1853163985071488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319054.556759:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319054.556763:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319054.556765:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319054.556769:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4297 00000100:00080000:2.0:1767319054.556771:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319054.556775:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319054.556804:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ad9f80 x1853163985071616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319054.556859:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319054.556862:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319054.556864:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319054.556868:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4297 00000100:00080000:2.0:1767319054.556870:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319054.556873:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319054.556885:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73dc00 x1853163985071872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319054.556900:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319054.556903:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319054.556905:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319054.556908:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4297 00000100:00080000:2.0:1767319054.556911:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319054.556914:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319054.556924:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f480 x1853163985072128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319054.557013:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985071744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319054.557090:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319054.557204:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319054.557211:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985072000:0@lo:400:kworker.0 00000100:00100000:3.0:1767319054.557234:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319054.557597:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985071744 00000100:00100000:2.0:1767319054.557605:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985071744 00000100:00080000:3.0:1767319054.557673:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319054.557684:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072000 00000100:00100000:2.0:1767319054.557688:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072000 00000100:00100000:2.0:1767319054.557723:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4389 00000100:00080000:3.0:1767319054.557726:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319054.557734:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985071744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319054.557761:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4390 00000100:00100000:3.0:1767319054.557773:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985072000:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319054.557807:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985071744:12345-0@lo:400:kworker.0 Request processed in 63us (704us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319054.557816:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4389 00000100:00100000:2.0:1767319054.557876:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985072000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319054.557887:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985071744:0@lo:400:kworker.0 00000100:00100000:0.0:1767319054.557986:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985071488:0@lo:38:kworker.0 00000100:00100000:3.0:1767319054.558013:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985072000:12345-0@lo:400:kworker.0 Request processed in 242us (776us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319054.558024:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4390 00000100:00100000:0.0:1767319054.558025:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319054.558081:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985071616:0@lo:38:kworker.0 00000100:00100000:0.0:1767319054.558118:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319054.558142:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985071872:0@lo:38:kworker.0 00000100:00100000:3.0:1767319054.558203:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985071488 00000100:00100000:3.0:1767319054.558208:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985071488 00000100:00100000:3.0:1767319054.558231:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4788 00000100:00100000:3.0:1767319054.558234:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985071488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319054.558252:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319054.558276:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985071488:12345-0@lo:38:kworker.0 Request processed in 41us (255us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319054.558280:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4788 00000100:00100000:0.0:1767319054.558369:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319054.558393:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985072128:0@lo:38:kworker.0 00000100:00100000:3.0:1767319054.558404:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985071872 00000100:00100000:3.0:1767319054.558410:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985071872 00000100:00100000:3.0:1767319054.558428:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4789 00000100:00100000:3.0:1767319054.558432:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985071872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319054.558442:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319054.558514:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985071616 00000100:00100000:1.0:1767319054.558525:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985071616 00000100:00100000:1.0:1767319054.558552:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1687 00000100:00100000:1.0:1767319054.558557:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985071616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319054.558569:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319054.558748:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319054.559871:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985071872:12345-0@lo:38:kworker.0 Request processed in 1432us (1702us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319054.559884:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4789 00000100:00080000:0.0:1767319054.559953:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319054.559960:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319054.559963:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985071488:0@lo:38:kworker.0 00000100:00080000:0.0:1767319054.559974:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319054.559978:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319054.559982:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985071872:0@lo:38:kworker.0 00000100:00100000:2.0:1767319054.559991:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072128 00000100:00100000:2.0:1767319054.559997:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072128 00000100:00100000:2.0:1767319054.560024:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4790 00000100:00100000:2.0:1767319054.560030:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985072128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319054.560043:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319054.560228:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985072128:12345-0@lo:38:kworker.0 Request processed in 196us (1483us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319054.560235:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4790 00000100:00100000:1.0:1767319054.563386:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985071616:12345-0@lo:38:kworker.0 Request processed in 4818us (5267us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319054.563399:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1687 00000100:00080000:0.0:1767319054.563446:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319054.563451:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319054.563454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985071616:0@lo:38:kworker.0 00000100:00080000:0.0:1767319054.563464:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319054.563466:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319054.563468:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985072128:0@lo:38:kworker.0 00010000:00010000:1.0:1767319055.197109:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319056.219451:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319057.243612:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319058.267406:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319059.164417:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985072256:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319059.164571:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.164692:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072256 00000100:00100000:2.0:1767319059.164702:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072256 00000100:00100000:2.0:1767319059.164786:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2056 00000100:00100000:2.0:1767319059.164796:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985072256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319059.164825:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319059.164868:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985072384:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319059.164899:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319059.165625:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072384 00000100:00100000:0.0:1767319059.165631:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072384 00000100:00100000:0.0:1767319059.165663:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2057 00000100:00100000:0.0:1767319059.165671:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985072384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319059.165690:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319059.165742:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985072384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (841us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319059.165749:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2057 00000100:00100000:0.0:1767319059.165795:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985072384:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319059.165828:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319059.165833:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319059.169626:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985072256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4822us (5046us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319059.169642:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2056 00000100:00100000:1.0:1767319059.171905:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985072256:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319059.291544:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319059.630030:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319059.630107:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319059.632471:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920539392 00000100:00100000:0.0:1767319059.632484:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920539392 00000100:00100000:0.0:1767319059.632613:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920539264 00000100:00100000:0.0:1767319059.632617:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920539264 00000100:00100000:0.0:1767319059.632641:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4391 00000100:00100000:0.0:1767319059.632653:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920539392:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319059.632743:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920539392:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (2721us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319059.632750:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4391 00000100:00100000:0.0:1767319059.632770:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4791 00000100:00100000:0.0:1767319059.632775:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920539264:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319059.632795:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319059.632821:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920539264:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (2713us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319059.632828:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4791 00000100:00080000:1.0:1767319059.636534:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319059.675464:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.675475:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319059.675478:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.675482:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.675486:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319059.675494:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.675498:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.675506:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319059.675549:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357ad80 x1853163985072512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319059.675576:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.675579:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.675581:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319059.675584:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.675587:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.675590:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319059.675603:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a300 x1853163985072640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319059.675650:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.675654:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.675655:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319059.675659:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.675661:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.675664:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319059.675681:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579c00 x1853163985072896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319059.675707:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.675710:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.675712:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319059.675715:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.675718:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.675721:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319059.675732:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985073152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319059.675800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985072512:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.675874:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.675935:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985072640:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.675960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.675983:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985072896:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.676005:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.676022:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985073152:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.676041:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.676075:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072512 00000100:00100000:2.0:1767319059.676082:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072512 00000100:00100000:2.0:1767319059.676113:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072896 00000100:00100000:2.0:1767319059.676116:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072896 00000100:00100000:2.0:1767319059.676129:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985073152 00000100:00100000:2.0:1767319059.676132:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985073152 00000100:00100000:2.0:1767319059.676151:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4792 00000100:00100000:2.0:1767319059.676158:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985072512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319059.676210:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319059.676253:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985072512:12345-0@lo:38:kworker.0 Request processed in 95us (379us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319059.676260:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4792 00000100:00100000:2.0:1767319059.676278:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4793 00000100:00100000:2.0:1767319059.676282:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985072896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319059.676292:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319059.676315:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985072896:12345-0@lo:38:kworker.0 Request processed in 31us (309us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319059.676320:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4793 00000100:00100000:2.0:1767319059.676332:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4794 00000100:00100000:2.0:1767319059.676336:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985073152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319059.676344:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319059.676364:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985073152:12345-0@lo:38:kworker.0 Request processed in 27us (323us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319059.676369:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4794 00000100:00080000:2.0:1767319059.676407:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319059.676411:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319059.676415:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985072512:0@lo:38:kworker.0 00000100:00080000:2.0:1767319059.676425:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319059.676428:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319059.676431:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985072896:0@lo:38:kworker.0 00000100:00080000:2.0:1767319059.676438:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319059.676441:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319059.676444:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985073152:0@lo:38:kworker.0 00000100:00100000:1.0:1767319059.677861:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985073024:0@lo:400:kworker.0 00000100:00100000:1.0:1767319059.677922:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319059.677974:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319059.677982:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985072768:0@lo:400:kworker.0 00000100:00100000:1.0:1767319059.678006:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319059.678049:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072640 00000100:00100000:1.0:1767319059.678057:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072640 00000100:00100000:1.0:1767319059.678094:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1688 00000100:00100000:1.0:1767319059.678100:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985072640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319059.678117:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319059.678160:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985072640:12345-0@lo:38:kworker.0 Request processed in 58us (2198us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319059.678285:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985073024 00000100:00100000:3.0:1767319059.678290:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985073024 00000100:00100000:3.0:1767319059.678330:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985072768 00000100:00100000:3.0:1767319059.678333:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985072768 00000100:00100000:3.0:1767319059.678351:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4392 00000100:00100000:3.0:1767319059.678357:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985073024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319059.678402:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985073024:12345-0@lo:400:kworker.0 Request processed in 45us (479us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319059.678409:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4392 00000100:00100000:3.0:1767319059.678425:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4393 00000100:00100000:3.0:1767319059.678431:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985072768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319059.678459:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985072768:12345-0@lo:400:kworker.0 Request processed in 29us (452us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319059.678465:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4393 00000100:00080000:3.0:1767319059.678525:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319059.678531:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319059.678535:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985072640:0@lo:38:kworker.0 00000100:00080000:3.0:1767319059.678562:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319059.678590:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985072768:0@lo:400:kworker.0 00000100:00100000:3.0:1767319059.678598:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985073024:0@lo:400:kworker.0 00000100:00100000:1.0:1767319059.678762:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1688 00010000:00010000:1.0:1767319060.315523:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319061.339415:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319062.364429:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319063.387543:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767319063.723975:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920539520 offset 0 length 520: 4 00000100:00100000:2.0:1767319063.724039:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319063.724084:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319063.724147:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920539648 00000100:00100000:3.0:1767319063.724161:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920539648 00000100:00100000:3.0:1767319063.724297:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920539776 00000100:00100000:3.0:1767319063.724301:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920539776 00000100:00080000:1.0:1767319063.724403:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319063.724544:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4394 00000100:00100000:2.0:1767319063.724660:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920539776:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319063.724785:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920539776:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 229us (698us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319063.724793:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4394 00000100:00100000:2.0:1767319063.724825:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4795 00000100:00100000:2.0:1767319063.724830:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920539648:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319063.724856:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319063.724886:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920539648:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (847us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319063.724894:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4795 00000100:00100000:0.0:1767319064.283507:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319064.283551:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985073280:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319064.283669:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319064.283741:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985073280 00000100:00100000:2.0:1767319064.283756:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985073280 00000100:00100000:2.0:1767319064.283842:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2058 00000100:00100000:2.0:1767319064.283852:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985073280:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319064.283888:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767319064.283891:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319064.283982:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985073280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 131us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319064.283990:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2058 00000100:00100000:3.0:1767319064.284085:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985073280:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319064.285127:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985073408:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319064.285215:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319064.285389:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985073408 00000100:00100000:1.0:1767319064.285397:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985073408 00000100:00100000:1.0:1767319064.285485:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2059 00000100:00100000:1.0:1767319064.285496:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985073408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319064.285529:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767319064.285608:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319064.286000:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985073408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 504us (783us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319064.286010:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2059 00000100:00100000:3.0:1767319064.286085:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985073408:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319064.411971:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319064.795348:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319039/real 1767319039] req@ffff8ce1a55a3480 x1853163985068032/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319064 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319064.795396:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a55a3480 x1853163985068032/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319064 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319064.795423:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319064.795433:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319064.795438:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985068032:0@lo:250:kworker.0 00000100:00080000:1.0:1767319064.797324:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319064.797338:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319064.797342:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319064.797352:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4287 00000100:00080000:1.0:1767319064.797358:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319064.797366:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319064.797425:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6d80 x1853163985073536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319064.797462:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319064.797466:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319064.797468:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319064.797471:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4307 00000100:00080000:1.0:1767319064.797474:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319064.797477:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319064.797495:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6a00 x1853163985073664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319064.797504:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319064.797508:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319064.797510:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319064.797513:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4307 00000100:00080000:1.0:1767319064.797516:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319064.797519:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319064.797797:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7b80 x1853163985073792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319064.797851:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319064.797855:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319064.797856:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319064.797860:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4307 00000100:00080000:1.0:1767319064.797862:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319064.797866:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319064.797877:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6680 x1853163985074048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319064.797896:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319064.797899:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319064.797901:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319064.797905:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4307 00000100:00080000:1.0:1767319064.797907:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319064.797913:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319064.797913:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985074176:0@lo:400:kworker.0 00000100:00100000:1.0:1767319064.797927:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adf800 x1853163985074304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319064.798100:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319064.798129:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985073920:0@lo:400:kworker.0 00000100:00100000:0.0:1767319064.798182:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319064.798314:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ade300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985074432:0@lo:400:kworker.0 00000100:00100000:2.0:1767319064.798351:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319064.798389:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319064.798396:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985074560:0@lo:400:kworker.0 00000100:00100000:0.0:1767319064.798419:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074176 00000100:00100000:2.0:1767319064.798424:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319064.798433:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074176 00000100:00100000:2.0:1767319064.798468:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074432 00000100:00100000:2.0:1767319064.798473:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074432 00000100:00100000:0.0:1767319064.798503:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985073920 00000100:00100000:0.0:1767319064.798508:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985073920 00000100:00100000:2.0:1767319064.798516:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074560 00000100:00100000:2.0:1767319064.798521:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074560 00000100:00100000:0.0:1767319064.798544:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4395 00000100:00100000:2.0:1767319064.798550:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2897 00000100:00100000:0.0:1767319064.798558:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985074176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319064.798563:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985074432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319064.798640:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985074432:12345-0@lo:400:kworker.0 Request processed in 80us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319064.798647:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985074176:12345-0@lo:400:kworker.0 Request processed in 92us (546us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319064.798653:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2897 00000100:00100000:0.0:1767319064.798664:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4395 00000100:00100000:2.0:1767319064.798680:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2898 00000100:00100000:2.0:1767319064.798694:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985074560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319064.798695:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4396 00000100:00100000:0.0:1767319064.798709:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985073920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319064.798736:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985074560:12345-0@lo:400:kworker.0 Request processed in 49us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319064.798745:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2898 00000100:00100000:0.0:1767319064.798752:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985073920:12345-0@lo:400:kworker.0 Request processed in 46us (586us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319064.798759:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4396 00000100:00080000:0.0:1767319064.798784:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319064.798806:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985073920:0@lo:400:kworker.0 00000100:00100000:0.0:1767319064.798831:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985074176:0@lo:400:kworker.0 00000100:00100000:2.0:1767319064.798851:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985074560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319064.798863:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ade300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985074432:0@lo:400:kworker.0 00000100:00100000:3.0:1767319064.800738:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985073536:0@lo:250:kworker.0 00000400:00000100:3.0:1767319064.800799:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985073536 offset 0 length 520: 4 00000100:00100000:3.0:1767319064.800834:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985073664:0@lo:38:kworker.0 00000100:00100000:3.0:1767319064.800864:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319064.800885:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985073792:0@lo:38:kworker.0 00000100:00100000:3.0:1767319064.800911:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319064.800940:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985074048:0@lo:38:kworker.0 00000100:00100000:3.0:1767319064.800963:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319064.800984:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985074304:0@lo:38:kworker.0 00000100:00100000:3.0:1767319064.801006:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319064.801047:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985073664 00000100:00100000:3.0:1767319064.801057:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985073664 00000100:00100000:3.0:1767319064.801091:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074048 00000100:00100000:3.0:1767319064.801094:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074048 00000100:00100000:3.0:1767319064.801108:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074304 00000100:00100000:3.0:1767319064.801110:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074304 00000100:00100000:3.0:1767319064.801122:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4796 00000100:00100000:3.0:1767319064.801129:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985073664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319064.801154:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319064.801222:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985073664:12345-0@lo:38:kworker.0 Request processed in 91us (356us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319064.801230:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4796 00000100:00100000:3.0:1767319064.801246:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4797 00000100:00100000:3.0:1767319064.801250:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985074048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319064.801258:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319064.801277:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985074048:12345-0@lo:38:kworker.0 Request processed in 26us (313us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319064.801282:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4797 00000100:00100000:3.0:1767319064.801290:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4798 00000100:00100000:3.0:1767319064.801294:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985074304:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319064.801301:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319064.801319:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985074304:12345-0@lo:38:kworker.0 Request processed in 24us (313us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319064.801324:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4798 00000100:00080000:3.0:1767319064.801373:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319064.801379:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319064.801383:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985073664:0@lo:38:kworker.0 00000100:00080000:3.0:1767319064.801392:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319064.801395:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319064.801398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985074048:0@lo:38:kworker.0 00000100:00080000:3.0:1767319064.801404:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319064.801408:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319064.801411:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985074304:0@lo:38:kworker.0 00000100:00100000:2.0:1767319064.801717:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985073792 00000100:00100000:2.0:1767319064.801723:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985073792 00000100:00100000:2.0:1767319064.801757:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1689 00000100:00100000:2.0:1767319064.801762:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985073792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319064.801776:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319064.801823:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985073792:12345-0@lo:38:kworker.0 Request processed in 59us (911us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319064.801830:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1689 00000100:00080000:1.0:1767319064.801857:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319064.801864:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319064.801869:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985073792:0@lo:38:kworker.0 00010000:00010000:0.0:1767319065.435512:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319066.459533:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319067.483534:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319068.507509:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319068.841860:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319068.841952:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920539904 00000100:00100000:2.0:1767319068.841967:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920539904 00000100:00100000:2.0:1767319068.842033:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4799 00000100:00100000:2.0:1767319068.842044:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920539904:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319068.842076:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319068.842222:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920539904:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 179us (377us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319068.842231:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4799 00000100:00100000:1.0:1767319068.843793:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319068.843854:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920540032 00000100:00100000:3.0:1767319068.843867:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920540032 00000100:00100000:3.0:1767319068.843965:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4397 00000100:00100000:3.0:1767319068.843977:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920540032:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319068.844089:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920540032:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 113us (308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319068.844098:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4397 00000100:00080000:0.0:1767319068.844220:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319069.404573:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319069.404612:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985074688:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319069.404703:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319069.405191:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074688 00000100:00100000:0.0:1767319069.405203:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074688 00000100:00100000:0.0:1767319069.405552:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2060 00000100:00100000:0.0:1767319069.405563:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985074688:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319069.405847:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319069.405886:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319069.405902:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985074816:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319069.405960:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319069.405998:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074816 00000100:00100000:2.0:1767319069.406003:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074816 00000100:00100000:2.0:1767319069.406064:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2061 00000100:00100000:2.0:1767319069.406071:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985074816:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319069.406097:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319069.406145:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985074816:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (184us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319069.406153:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2061 00002000:00000024:0.0:1767319069.406195:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319069.406234:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985074816:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319069.406697:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985074688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1135us (1992us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319069.406710:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2060 00000100:00100000:3.0:1767319069.406995:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985074688:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319069.407198:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319069.531504:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319069.916274:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319069.916289:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319069.916293:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319069.916298:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319069.916302:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319069.916313:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4313 00000100:00080000:3.0:1767319069.916320:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319069.916329:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319069.916460:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0380 x1853163985074944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319069.916540:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985074944:0@lo:38:kworker.0 00000100:00100000:1.0:1767319069.916633:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319069.916748:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319069.916754:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319069.916756:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319069.916760:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4313 00000100:00080000:3.0:1767319069.916763:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319069.916767:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319069.916784:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985075072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319069.916822:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319069.916826:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319069.916828:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319069.916831:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4313 00000100:00080000:3.0:1767319069.916834:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319069.916838:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319069.916850:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb480 x1853163985075328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319069.916866:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319069.916869:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319069.916871:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319069.916874:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4313 00000100:00080000:3.0:1767319069.916877:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319069.916880:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319069.916907:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985075584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319069.916960:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985075456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319069.916995:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985075840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319069.917039:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319069.917102:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319069.917109:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985075712:0@lo:400:kworker.0 00000100:00100000:3.0:1767319069.917397:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319069.917468:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075840 00000100:00100000:3.0:1767319069.917477:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075840 00000100:00100000:3.0:1767319069.917539:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075712 00000100:00100000:3.0:1767319069.917543:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075712 00000100:00100000:3.0:1767319069.917567:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2899 00000100:00100000:3.0:1767319069.917576:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985075840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319069.917636:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985075840:12345-0@lo:400:kworker.0 Request processed in 61us (596us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319069.917644:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2899 00000100:00100000:3.0:1767319069.917662:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2900 00000100:00100000:3.0:1767319069.917668:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985075712:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319069.917696:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985075712:12345-0@lo:400:kworker.0 Request processed in 28us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319069.917701:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2900 00000100:00100000:3.0:1767319069.917743:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985075712:0@lo:400:kworker.0 00000100:00100000:3.0:1767319069.917754:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985075840:0@lo:400:kworker.0 00000100:00100000:1.0:1767319069.917906:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985075072:0@lo:38:kworker.0 00000100:00100000:1.0:1767319069.917939:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319069.917965:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985075328:0@lo:38:kworker.0 00000100:00100000:1.0:1767319069.917990:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319069.918335:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985075200:0@lo:400:kworker.0 00000100:00100000:0.0:1767319069.918398:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075072 00000100:00100000:0.0:1767319069.918417:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075072 00000100:00100000:0.0:1767319069.918455:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1690 00000100:00100000:0.0:1767319069.918460:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985075072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319069.918482:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319069.918529:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985075072:12345-0@lo:38:kworker.0 Request processed in 67us (589us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319069.918536:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1690 00000100:00100000:1.0:1767319069.919275:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985075584:0@lo:38:kworker.0 00000100:00100000:1.0:1767319069.919316:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319069.919389:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985074944 00000100:00100000:2.0:1767319069.919397:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985074944 00000100:00100000:2.0:1767319069.919428:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075328 00000100:00100000:2.0:1767319069.919431:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075328 00000100:00100000:2.0:1767319069.919450:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4800 00000100:00100000:2.0:1767319069.919457:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985074944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319069.919473:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319069.919528:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985074944:12345-0@lo:38:kworker.0 Request processed in 70us (2895us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319069.919534:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4800 00000100:00100000:2.0:1767319069.919549:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4801 00000100:00100000:2.0:1767319069.919553:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985075328:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319069.919562:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319069.919589:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985075328:12345-0@lo:38:kworker.0 Request processed in 35us (1598us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319069.919594:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4801 00000100:00100000:2.0:1767319069.919771:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075584 00000100:00100000:2.0:1767319069.919776:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075584 00000100:00100000:2.0:1767319069.919795:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4802 00000100:00100000:2.0:1767319069.919799:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985075584:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319069.919810:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319069.920020:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319069.920076:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075456 00000100:00100000:0.0:1767319069.920080:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075456 00000100:00100000:0.0:1767319069.920126:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4398 00000100:00100000:0.0:1767319069.920144:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985075456:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319069.920213:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985075456:12345-0@lo:400:kworker.0 Request processed in 81us (194us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319069.920220:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4398 00000100:00080000:0.0:1767319069.920240:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319069.920254:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319069.920287:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075200 00000100:00100000:0.0:1767319069.920291:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075200 00000100:00100000:0.0:1767319069.920315:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4399 00000100:00100000:0.0:1767319069.920321:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985075200:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319069.920357:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985075200:12345-0@lo:400:kworker.0 Request processed in 35us (101us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319069.920366:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4399 00000100:00080000:0.0:1767319069.920380:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319069.920410:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985075200:0@lo:400:kworker.0 00000100:00100000:0.0:1767319069.920440:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985075456:0@lo:400:kworker.0 00000100:00100000:2.0:1767319069.920742:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985075584:12345-0@lo:38:kworker.0 Request processed in 939us (1423us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319069.920749:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4802 00000100:00080000:1.0:1767319069.920812:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319069.920819:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319069.920822:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985075584:0@lo:38:kworker.0 00000100:00080000:1.0:1767319069.920835:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319069.920838:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319069.920841:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985074944:0@lo:38:kworker.0 00000100:00080000:1.0:1767319069.920868:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319069.920872:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319069.920875:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985075072:0@lo:38:kworker.0 00000100:00080000:1.0:1767319069.921013:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319069.921018:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319069.921087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985075328:0@lo:38:kworker.0 00010000:00010000:2.0:1767319070.560695:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319071.579461:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319072.604009:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319073.627504:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319073.965259:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319073.965372:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319073.966110:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920540288 00000100:00100000:1.0:1767319073.966124:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920540288 00000100:00100000:1.0:1767319073.966422:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920540160 00000100:00100000:1.0:1767319073.966426:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920540160 00000100:00100000:1.0:1767319073.966467:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4400 00000100:00100000:1.0:1767319073.966478:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920540288:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319073.966580:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920540288:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 104us (1336us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319073.966592:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4400 00000100:00100000:1.0:1767319073.966612:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4803 00000100:00100000:1.0:1767319073.966616:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920540160:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319073.966642:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319073.966666:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920540160:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (1295us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319073.966672:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4803 00000100:00080000:0.0:1767319073.967528:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319074.524493:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985075968:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319074.524597:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319074.524653:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985075968 00000100:00100000:1.0:1767319074.524667:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985075968 00000100:00100000:1.0:1767319074.524774:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2062 00000100:00100000:1.0:1767319074.524786:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985075968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319074.524830:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319074.524919:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985075968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 135us (324us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319074.524928:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2062 00000100:00100000:3.0:1767319074.525090:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985075968:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319074.525199:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319074.525328:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985076096:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319074.525373:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319074.525474:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076096 00000100:00100000:2.0:1767319074.525481:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076096 00000100:00100000:2.0:1767319074.525556:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2063 00000100:00100000:2.0:1767319074.525567:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985076096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319074.525594:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319074.525636:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985076096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (263us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319074.525643:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2063 00000100:00100000:2.0:1767319074.525675:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985076096:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319074.526914:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319074.655061:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319075.037743:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319075.037752:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319075.037756:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319075.037760:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319075.037765:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319075.037773:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4318 00000100:00080000:0.0:1767319075.037778:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319075.037784:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319075.037841:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ad9f80 x1853163985076224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319075.037871:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319075.037875:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319075.037877:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319075.037900:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4318 00000100:00080000:0.0:1767319075.037902:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319075.037906:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319075.037922:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3480 x1853163985076352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319075.037959:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985076224:0@lo:38:kworker.0 00000100:00080000:0.0:1767319075.037981:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319075.037984:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319075.037986:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319075.037990:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4318 00000100:00080000:0.0:1767319075.037992:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319075.037995:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319075.038008:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0a80 x1853163985076608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319075.038022:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319075.038025:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319075.038027:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319075.038030:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4318 00000100:00100000:3.0:1767319075.038030:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319075.038032:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319075.038035:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319075.038043:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0e00 x1853163985076864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319075.038088:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985076352:0@lo:38:kworker.0 00000100:00100000:0.0:1767319075.038104:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985076992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319075.038116:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319075.038133:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319075.038150:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985076608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319075.038155:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319075.038160:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985077120:0@lo:400:kworker.0 00000100:00100000:3.0:1767319075.038187:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319075.038198:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319075.038203:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985076864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319075.038221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319075.038226:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076992 00000100:00100000:0.0:1767319075.038233:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076992 00000100:00100000:3.0:1767319075.038246:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076352 00000100:00100000:3.0:1767319075.038249:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076352 00000100:00100000:0.0:1767319075.038270:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985077120 00000100:00100000:0.0:1767319075.038272:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985077120 00000100:00100000:3.0:1767319075.038276:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1691 00000100:00100000:3.0:1767319075.038283:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985076352:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319075.038288:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2901 00000100:00100000:0.0:1767319075.038293:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985076992:12345-0@lo:400:kworker.0 00010000:00020000:3.0:1767319075.038302:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319075.038335:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985076992:12345-0@lo:400:kworker.0 Request processed in 40us (200us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319075.038335:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985076352:12345-0@lo:38:kworker.0 Request processed in 50us (217us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319075.038342:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2901 00000100:00100000:3.0:1767319075.038344:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1691 00000100:00100000:0.0:1767319075.038357:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2902 00000100:00100000:0.0:1767319075.038362:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985077120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319075.038362:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076224 00000100:00100000:3.0:1767319075.038365:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076224 00000100:00100000:3.0:1767319075.038381:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076608 00000100:00100000:3.0:1767319075.038384:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076608 00000100:00100000:0.0:1767319075.038386:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985077120:12345-0@lo:400:kworker.0 Request processed in 25us (187us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319075.038392:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2902 00000100:00100000:3.0:1767319075.038397:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076864 00000100:00100000:3.0:1767319075.038399:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076864 00000100:00100000:3.0:1767319075.038423:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4804 00000100:00100000:3.0:1767319075.038427:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985076224:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319075.038433:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985077120:0@lo:400:kworker.0 00010000:00020000:3.0:1767319075.038436:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319075.038443:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985076992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319075.038458:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985076224:12345-0@lo:38:kworker.0 Request processed in 29us (429us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319075.038463:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4804 00000100:00100000:3.0:1767319075.038474:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4805 00000100:00100000:3.0:1767319075.038478:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985076608:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319075.038486:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4806 00010000:00020000:3.0:1767319075.038486:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319075.038490:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985076864:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319075.038498:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319075.038506:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985076608:12345-0@lo:38:kworker.0 Request processed in 27us (318us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319075.038511:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4805 00000100:00100000:0.0:1767319075.038516:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985076864:12345-0@lo:38:kworker.0 Request processed in 26us (295us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319075.038521:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4806 00000100:00080000:3.0:1767319075.038542:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319075.038547:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319075.038550:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985076608:0@lo:38:kworker.0 00000100:00080000:3.0:1767319075.038557:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319075.038560:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319075.038562:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985076864:0@lo:38:kworker.0 00000100:00080000:3.0:1767319075.038572:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319075.038575:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319075.038577:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985076224:0@lo:38:kworker.0 00000100:00080000:3.0:1767319075.038583:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319075.038586:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319075.038589:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985076352:0@lo:38:kworker.0 00000100:00100000:2.0:1767319075.040467:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985076480:0@lo:400:kworker.0 00000100:00100000:2.0:1767319075.040548:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319075.040603:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319075.040610:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985076736:0@lo:400:kworker.0 00000100:00100000:2.0:1767319075.040631:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319075.040676:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076480 00000100:00100000:2.0:1767319075.040683:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076480 00000100:00100000:2.0:1767319075.040745:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985076736 00000100:00100000:2.0:1767319075.040749:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985076736 00000100:00100000:2.0:1767319075.040770:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4401 00000100:00100000:2.0:1767319075.040778:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985076480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319075.040825:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985076480:12345-0@lo:400:kworker.0 Request processed in 47us (276us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319075.040833:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4401 00000100:00100000:2.0:1767319075.040848:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4402 00000100:00100000:2.0:1767319075.040854:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985076736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319075.040892:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985076736:12345-0@lo:400:kworker.0 Request processed in 38us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319075.040898:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4402 00000100:00080000:2.0:1767319075.040919:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319075.040948:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985076736:0@lo:400:kworker.0 00000100:00100000:2.0:1767319075.040957:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985076480:0@lo:400:kworker.0 00010000:00010000:2.0:1767319075.676503:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319076.700688:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319077.724097:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319078.749898:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319079.084101:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319079.084450:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920540416 00000100:00100000:0.0:1767319079.084461:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920540416 00000100:00100000:0.0:1767319079.084509:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4807 00000100:00100000:0.0:1767319079.084519:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920540416:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319079.084543:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319079.085784:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920540416:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1259us (1685us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319079.085796:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4807 00000100:00100000:1.0:1767319079.087874:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319079.088055:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920540544 00000100:00100000:0.0:1767319079.088066:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920540544 00000100:00100000:0.0:1767319079.088225:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4403 00000100:00100000:0.0:1767319079.088234:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920540544:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319079.088317:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920540544:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (450us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319079.088325:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4403 00000100:00080000:2.0:1767319079.089387:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319079.643840:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319079.643865:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985077248:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319079.643947:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319079.644068:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985077248 00000100:00100000:2.0:1767319079.644077:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985077248 00000100:00080000:1.0:1767319079.644143:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319079.644153:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2064 00000100:00100000:2.0:1767319079.644185:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985077248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319079.644218:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319079.644224:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985077376:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319079.644262:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319079.644273:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985077248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 110us (324us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319079.644282:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2064 00000100:00100000:2.0:1767319079.644302:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985077376 00000100:00100000:2.0:1767319079.644305:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985077376 00000100:00100000:0.0:1767319079.644344:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2065 00000100:00100000:0.0:1767319079.644355:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985077376:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319079.644359:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985077248:0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319079.644373:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767319079.644383:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319079.644403:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985077376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (142us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319079.644410:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2065 00000100:00100000:1.0:1767319079.644420:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985077376:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319079.772943:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319080.155578:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319080.155588:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319080.155591:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319080.155595:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319080.155600:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319080.155632:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4323 00000100:00080000:3.0:1767319080.155637:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319080.155644:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319080.155682:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939180 x1853163985077504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319080.155697:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319080.155701:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319080.155702:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319080.155706:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4323 00000100:00080000:3.0:1767319080.155708:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319080.155711:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319080.155721:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985077632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319080.155752:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319080.155756:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319080.155759:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319080.155763:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4323 00000100:00080000:3.0:1767319080.155766:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319080.155769:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319080.155774:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319080.155782:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938a80 x1853163985077888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319080.155791:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985077760:0@lo:400:kworker.0 00000100:00080000:3.0:1767319080.155802:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319080.155806:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319080.155808:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319080.155812:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4323 00000100:00080000:3.0:1767319080.155815:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319080.155818:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319080.155838:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593bb80 x1853163985078144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319080.155861:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319080.155899:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985078272:0@lo:400:kworker.0 00000100:00100000:2.0:1767319080.155934:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985078016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319080.155939:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319080.155966:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319080.155969:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319080.155975:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985078400:0@lo:400:kworker.0 00000100:00100000:2.0:1767319080.156004:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985077760 00000100:00100000:3.0:1767319080.156005:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319080.156013:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985077760 00000100:00100000:3.0:1767319080.156036:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078272 00000100:00100000:3.0:1767319080.156041:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078272 00000100:00100000:2.0:1767319080.156063:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078016 00000100:00100000:2.0:1767319080.156068:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078016 00000100:00100000:3.0:1767319080.156081:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078400 00000100:00100000:3.0:1767319080.156094:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078400 00000100:00100000:3.0:1767319080.156114:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2903 00000100:00100000:3.0:1767319080.156122:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985078272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319080.156920:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4404 00000100:00100000:2.0:1767319080.156930:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985077760:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319080.157582:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2904 00000100:00100000:0.0:1767319080.157595:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985078400:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319080.157700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985077504:0@lo:38:kworker.0 00000100:00100000:2.0:1767319080.160465:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985077760:12345-0@lo:400:kworker.0 Request processed in 3527us (4594us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319080.160480:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4404 00000100:00100000:2.0:1767319080.160519:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4405 00000100:00100000:2.0:1767319080.160533:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985078016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319080.160580:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985078016:12345-0@lo:400:kworker.0 Request processed in 54us (4612us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319080.160586:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4405 00000100:00100000:2.0:1767319080.160638:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985078016:0@lo:400:kworker.0 00000100:00100000:2.0:1767319080.160648:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985077760:0@lo:400:kworker.0 00000100:00080000:2.0:1767319080.160814:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319080.162035:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985078272:12345-0@lo:400:kworker.0 Request processed in 5907us (6092us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319080.162048:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2903 00000100:00100000:0.0:1767319080.163403:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985078400:12345-0@lo:400:kworker.0 Request processed in 5804us (7390us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319080.163417:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2904 00000100:00100000:2.0:1767319080.164877:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985078272:0@lo:400:kworker.0 00000100:00100000:2.0:1767319080.164907:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985078400:0@lo:400:kworker.0 00000100:00100000:1.0:1767319080.170000:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319080.170086:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985077632:0@lo:38:kworker.0 00000100:00100000:1.0:1767319080.170121:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319080.170144:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985077888:0@lo:38:kworker.0 00000100:00100000:1.0:1767319080.170199:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319080.170210:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985077504 00000100:00100000:1.0:1767319080.170220:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985078144:0@lo:38:kworker.0 00000100:00100000:2.0:1767319080.170220:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985077504 00000100:00100000:1.0:1767319080.170247:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319080.170337:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985077632 00000100:00100000:0.0:1767319080.170343:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985077632 00000100:00100000:0.0:1767319080.170375:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1692 00000100:00100000:0.0:1767319080.170383:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985077632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319080.170407:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319080.170447:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985077632:12345-0@lo:38:kworker.0 Request processed in 64us (325us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319080.170454:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1692 00000100:00080000:1.0:1767319080.173436:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319080.173448:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319080.173452:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985077632:0@lo:38:kworker.0 00000100:00100000:2.0:1767319080.175476:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985077888 00000100:00100000:2.0:1767319080.175490:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985077888 00000100:00100000:2.0:1767319080.175542:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078144 00000100:00100000:2.0:1767319080.175546:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078144 00000100:00100000:2.0:1767319080.175571:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4808 00000100:00100000:2.0:1767319080.175578:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985077504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319080.175597:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319080.175703:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4809 00000100:00100000:1.0:1767319080.175709:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985077888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319080.175720:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319080.175787:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4810 00000100:00100000:0.0:1767319080.175793:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985078144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319080.175804:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319080.176461:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985077888:12345-0@lo:38:kworker.0 Request processed in 749us (6261us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319080.176469:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4809 00000100:00080000:1.0:1767319080.176513:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319080.176518:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319080.176522:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985077504:0@lo:38:kworker.0 00000100:00080000:1.0:1767319080.176532:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319080.176535:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319080.176538:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985077888:0@lo:38:kworker.0 00000100:00100000:0.0:1767319080.177385:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985078144:12345-0@lo:38:kworker.0 Request processed in 1587us (7135us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319080.177395:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4810 00000100:00080000:1.0:1767319080.177454:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319080.177458:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319080.177462:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985078144:0@lo:38:kworker.0 00000100:00100000:2.0:1767319080.182227:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985077504:12345-0@lo:38:kworker.0 Request processed in 6639us (12224us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319080.182242:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4808 00010000:00010000:1.0:1767319080.796860:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319081.819444:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319082.844957:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319083.867680:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319084.204600:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319084.204707:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319084.209712:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920540672 00000100:00100000:0.0:1767319084.209728:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920540672 00000100:00100000:0.0:1767319084.209778:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920540800 00000100:00100000:0.0:1767319084.209781:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920540800 00000100:00100000:0.0:1767319084.209880:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4406 00000100:00100000:0.0:1767319084.209891:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920540800:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319084.209968:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920540800:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (5261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319084.209976:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4406 00000100:00100000:2.0:1767319084.211376:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4811 00000100:00100000:2.0:1767319084.211387:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ade300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920540672:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319084.211409:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319084.211457:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ade300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920540672:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 70us (6866us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319084.211464:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4811 00000100:00080000:3.0:1767319084.213406:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319084.763744:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319084.763767:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985078528:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319084.763829:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319084.764021:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078528 00000100:00100000:2.0:1767319084.764029:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078528 00000100:00100000:2.0:1767319084.764094:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2066 00000100:00100000:2.0:1767319084.764103:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985078528:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319084.764133:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319084.764214:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985078528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 112us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319084.764222:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2066 00000100:00100000:3.0:1767319084.764272:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319084.764283:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985078656:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319084.764314:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319084.764375:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319084.764437:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985078528:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319084.765243:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078656 00000100:00100000:2.0:1767319084.765250:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078656 00000100:00100000:2.0:1767319084.765302:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2067 00000100:00100000:2.0:1767319084.765311:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985078656:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319084.765339:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767319084.765365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319084.765395:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985078656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (1079us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319084.765403:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2067 00000100:00100000:1.0:1767319084.765454:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985078656:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319084.891922:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319085.275881:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319085.275893:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319085.275896:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319085.275900:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319085.275904:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319085.275913:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4328 00000100:00080000:2.0:1767319085.275918:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319085.275924:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319085.275964:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939180 x1853163985078784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319085.275990:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319085.275994:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319085.275996:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319085.275999:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4328 00000100:00080000:2.0:1767319085.276002:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319085.276005:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319085.276017:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985078912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319085.276062:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319085.276067:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319085.276069:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319085.276072:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4328 00000100:00080000:2.0:1767319085.276075:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319085.276078:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319085.276090:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a300 x1853163985079168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319085.276105:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319085.276108:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319085.276109:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319085.276112:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4328 00000100:00080000:2.0:1767319085.276114:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319085.276117:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319085.276129:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939f80 x1853163985079424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319085.276196:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985079296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319085.276200:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985078784:0@lo:38:kworker.0 00000100:00100000:3.0:1767319085.276288:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985079040:0@lo:400:kworker.0 00000100:00100000:1.0:1767319085.279643:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319085.279720:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985078912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319085.279750:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319085.279772:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985079168:0@lo:38:kworker.0 00000100:00100000:1.0:1767319085.279785:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319085.279795:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985079424:0@lo:38:kworker.0 00000100:00100000:1.0:1767319085.279809:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319085.279844:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078784 00000100:00100000:1.0:1767319085.279850:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078784 00000100:00100000:1.0:1767319085.279872:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985079168 00000100:00100000:1.0:1767319085.279873:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985079168 00000100:00100000:1.0:1767319085.279896:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985079424 00000100:00100000:1.0:1767319085.280216:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985079424 00000100:00100000:1.0:1767319085.280255:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4812 00000100:00100000:1.0:1767319085.280262:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985078784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319085.280282:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319085.280415:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4813 00000100:00100000:3.0:1767319085.280421:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985079168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319085.280432:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319085.280476:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985079168:12345-0@lo:38:kworker.0 Request processed in 54us (689us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319085.280483:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4813 00000100:00100000:3.0:1767319085.280502:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4814 00000100:00100000:3.0:1767319085.280505:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985079424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319085.280514:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319085.280533:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985079424:12345-0@lo:38:kworker.0 Request processed in 26us (722us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319085.280539:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4814 00000100:00100000:3.0:1767319085.280559:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319085.280577:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985079040 00000100:00100000:3.0:1767319085.280580:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985079040 00000100:00100000:3.0:1767319085.280613:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4407 00000100:00100000:3.0:1767319085.280618:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985079040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319085.280651:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985079040:12345-0@lo:400:kworker.0 Request processed in 33us (91us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319085.280657:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4407 00000100:00100000:3.0:1767319085.280691:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985079040:0@lo:400:kworker.0 00000100:00080000:3.0:1767319085.280743:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319085.280756:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985078912 00000100:00100000:3.0:1767319085.280759:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985078912 00000100:00100000:3.0:1767319085.280779:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1693 00000100:00100000:3.0:1767319085.280784:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985078912:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319085.280793:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319085.280819:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985078912:12345-0@lo:38:kworker.0 Request processed in 34us (1066us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319085.280824:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1693 00000100:00080000:3.0:1767319085.280862:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319085.280867:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319085.280871:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985078912:0@lo:38:kworker.0 00000100:00080000:3.0:1767319085.280879:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319085.280882:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319085.280884:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985079168:0@lo:38:kworker.0 00000100:00080000:3.0:1767319085.280891:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319085.280894:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319085.280896:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985079424:0@lo:38:kworker.0 00000100:00100000:1.0:1767319085.282366:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985078784:12345-0@lo:38:kworker.0 Request processed in 2095us (2722us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319085.282378:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4812 00000100:00080000:3.0:1767319085.282971:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319085.282978:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319085.282982:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985078784:0@lo:38:kworker.0 00000100:00100000:0.0:1767319085.284259:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319085.284571:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985079296 00000100:00100000:2.0:1767319085.284579:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985079296 00000100:00100000:2.0:1767319085.284646:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4408 00000100:00100000:2.0:1767319085.284655:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985079296:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319085.284726:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985079296:12345-0@lo:400:kworker.0 Request processed in 73us (471us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319085.284733:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4408 00000100:00100000:1.0:1767319085.284859:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985079296:0@lo:400:kworker.0 00000100:00080000:3.0:1767319085.284920:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319085.915404:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319086.939541:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319087.965067:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319088.298577:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319088.298617:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920540928 offset 0 length 520: 4 00000100:00100000:0.0:1767319088.299027:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541184 00000100:00100000:0.0:1767319088.299037:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541184 00000100:00100000:0.0:1767319088.299116:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4409 00000100:00100000:0.0:1767319088.299127:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920541184:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319088.299249:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920541184:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (680us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319088.299258:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4409 00000100:00080000:3.0:1767319088.299648:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319088.301510:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319088.301651:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541056 00000100:00100000:0.0:1767319088.301659:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541056 00000100:00100000:0.0:1767319088.301700:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4815 00000100:00100000:0.0:1767319088.301708:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920541056:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319088.301728:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319088.301783:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920541056:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 74us (280us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319088.301791:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4815 00010000:00010000:2.0:1767319088.987517:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319089.371296:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319064/real 1767319064] req@ffff8ce1a61c6d80 x1853163985073536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319089 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319089.371330:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61c6d80 x1853163985073536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319089 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319089.371351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319089.371378:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319089.371382:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985073536:0@lo:250:kworker.0 00000100:00080000:3.0:1767319089.373082:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319089.373092:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319089.373095:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319089.373101:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4313 00000100:00080000:3.0:1767319089.373106:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319089.373112:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319089.373154:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb100 x1853163985079552/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319089.373209:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319089.373213:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319089.373223:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319089.373227:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4333 00000100:00080000:3.0:1767319089.373229:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319089.373233:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319089.373249:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985079680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319089.373258:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319089.373261:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319089.373263:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319089.373266:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4333 00000100:00080000:3.0:1767319089.373268:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319089.373271:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319089.373281:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9880 x1853163985079808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319089.373328:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319089.373331:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319089.373333:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319089.373336:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4333 00000100:00080000:3.0:1767319089.373338:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319089.373341:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319089.373351:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612e680 x1853163985080064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319089.373365:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319089.373368:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319089.373369:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319089.373372:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4333 00000100:00080000:3.0:1767319089.373374:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319089.373376:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319089.373384:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ca80 x1853163985080320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319089.373434:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985080448:0@lo:400:kworker.0 00000100:00100000:1.0:1767319089.373624:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985080576:0@lo:400:kworker.0 00000100:00100000:1.0:1767319089.373658:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985079552:0@lo:250:kworker.0 00000400:00000100:1.0:1767319089.373677:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985079552 offset 0 length 520: 4 00000100:00100000:1.0:1767319089.373721:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985079680:0@lo:38:kworker.0 00000100:00100000:1.0:1767319089.373744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319089.373762:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985079808:0@lo:38:kworker.0 00000100:00100000:1.0:1767319089.373782:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319089.373799:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985080064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319089.373819:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319089.373833:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985080320:0@lo:38:kworker.0 00000100:00100000:1.0:1767319089.373849:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319089.373884:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985079808 00000100:00100000:1.0:1767319089.373890:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985079808 00000100:00100000:1.0:1767319089.373923:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1694 00000100:00100000:1.0:1767319089.373930:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985079808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319089.373949:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319089.373998:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985079808:12345-0@lo:38:kworker.0 Request processed in 67us (214us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319089.374005:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1694 00000100:00080000:1.0:1767319089.374042:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319089.374046:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319089.374049:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985079808:0@lo:38:kworker.0 00000100:00100000:2.0:1767319089.374285:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985079680 00000100:00100000:2.0:1767319089.374290:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985079680 00000100:00100000:2.0:1767319089.374319:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080064 00000100:00100000:2.0:1767319089.374322:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080064 00000100:00100000:2.0:1767319089.374337:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080320 00000100:00100000:2.0:1767319089.374340:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080320 00000100:00100000:2.0:1767319089.374355:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4816 00000100:00100000:2.0:1767319089.374360:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985079680:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319089.374370:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319089.374400:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985079680:12345-0@lo:38:kworker.0 Request processed in 39us (655us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319089.374406:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4816 00000100:00100000:2.0:1767319089.374418:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4817 00000100:00100000:2.0:1767319089.374422:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985080064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319089.374431:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319089.374451:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985080064:12345-0@lo:38:kworker.0 Request processed in 27us (631us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319089.374456:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4817 00000100:00100000:2.0:1767319089.374466:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4818 00000100:00100000:2.0:1767319089.374469:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985080320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319089.374477:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319089.374497:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985080320:12345-0@lo:38:kworker.0 Request processed in 26us (647us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319089.374503:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4818 00000100:00100000:2.0:1767319089.374540:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985079936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319089.374563:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319089.374585:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319089.374590:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985080192:0@lo:400:kworker.0 00000100:00100000:2.0:1767319089.374610:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319089.374630:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985079936 00000100:00100000:2.0:1767319089.374634:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985079936 00000100:00100000:2.0:1767319089.374668:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080192 00000100:00100000:2.0:1767319089.374672:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080192 00000100:00100000:2.0:1767319089.374687:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4410 00000100:00100000:2.0:1767319089.374692:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985079936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319089.374729:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985079936:12345-0@lo:400:kworker.0 Request processed in 36us (165us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319089.374735:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4410 00000100:00100000:2.0:1767319089.374746:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4411 00000100:00100000:2.0:1767319089.374751:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985080192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319089.374775:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985080192:12345-0@lo:400:kworker.0 Request processed in 24us (165us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319089.374781:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4411 00000100:00080000:2.0:1767319089.374796:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319089.374817:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985080192:0@lo:400:kworker.0 00000100:00100000:2.0:1767319089.374826:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985079936:0@lo:400:kworker.0 00000100:00100000:3.0:1767319089.375135:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319089.375210:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319089.375396:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080448 00000100:00100000:3.0:1767319089.375401:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080448 00000100:00100000:3.0:1767319089.375432:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080576 00000100:00100000:3.0:1767319089.375435:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080576 00000100:00100000:3.0:1767319089.375454:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2905 00000100:00100000:3.0:1767319089.375460:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985080448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319089.375508:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985080448:12345-0@lo:400:kworker.0 Request processed in 48us (1994us total) trans 0 rc 0/0 00000100:00080000:0.0:1767319089.377047:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319089.377053:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319089.377098:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985080448:0@lo:400:kworker.0 00000100:00100000:3.0:1767319089.377623:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2905 00000100:00100000:3.0:1767319089.377652:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2906 00000100:00100000:3.0:1767319089.377665:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985080576:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319089.377742:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985080576:12345-0@lo:400:kworker.0 Request processed in 84us (2532us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319089.377749:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2906 00000100:00080000:1.0:1767319089.377841:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319089.377848:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319089.377852:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985079680:0@lo:38:kworker.0 00000100:00080000:1.0:1767319089.377862:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319089.377866:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319089.377869:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985080064:0@lo:38:kworker.0 00000100:00080000:1.0:1767319089.377878:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319089.377882:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319089.377885:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985080320:0@lo:38:kworker.0 00000100:00100000:0.0:1767319089.379207:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985080576:0@lo:400:kworker.0 00000100:00100000:3.0:1767319089.883426:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985080704:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319089.883503:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319089.883610:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080704 00000100:00100000:2.0:1767319089.883619:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080704 00000100:00100000:2.0:1767319089.883679:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2068 00000100:00100000:2.0:1767319089.883689:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985080704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319089.883719:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319089.883773:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985080704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319089.883780:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2068 00000100:00100000:1.0:1767319089.883865:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985080704:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319089.884758:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319089.884770:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985080832:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319089.884812:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319089.884893:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080832 00000100:00100000:2.0:1767319089.884899:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080832 00000100:00100000:2.0:1767319089.884935:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2069 00000100:00100000:2.0:1767319089.884942:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb7c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985080832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319089.884964:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319089.885125:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb7c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985080832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 182us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319089.885144:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2069 00000100:00100000:3.0:1767319089.885472:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985080832:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319090.011505:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319091.039182:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319092.059536:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319093.085323:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319093.418422:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319093.418591:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319093.418765:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541440 00000100:00100000:0.0:1767319093.418774:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541440 00000100:00100000:0.0:1767319093.418845:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541312 00000100:00100000:0.0:1767319093.418847:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541312 00000100:00100000:0.0:1767319093.418867:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4412 00000100:00100000:0.0:1767319093.418876:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920541440:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319093.418939:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920541440:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 63us (526us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319093.418946:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4412 00000100:00100000:0.0:1767319093.418962:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4819 00000100:00100000:0.0:1767319093.418966:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920541312:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319093.418985:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319093.419004:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920541312:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 37us (414us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319093.419010:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4819 00000100:00080000:2.0:1767319093.419197:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319094.114661:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319094.491544:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319094.491554:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319094.491557:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319094.491561:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319094.491565:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319094.491573:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4337 00000100:00080000:2.0:1767319094.491577:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319094.491584:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319094.491621:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938380 x1853163985080960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319094.491649:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319094.491653:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319094.491654:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319094.491658:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4337 00000100:00080000:2.0:1767319094.491661:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319094.491664:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319094.491675:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985081088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319094.491711:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319094.491714:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319094.491716:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319094.491719:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4337 00000100:00080000:2.0:1767319094.491722:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319094.491725:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319094.491734:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938a80 x1853163985081344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319094.491749:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319094.491752:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319094.491753:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319094.491757:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4337 00000100:00080000:2.0:1767319094.491759:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319094.491763:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319094.491771:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7100 x1853163985081600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319094.491854:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985081856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319094.491915:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319094.491975:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319094.491981:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985081728:0@lo:400:kworker.0 00000100:00100000:1.0:1767319094.492000:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319094.492028:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081856 00000100:00100000:1.0:1767319094.492034:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081856 00000100:00100000:1.0:1767319094.492085:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081728 00000100:00100000:1.0:1767319094.492089:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081728 00000100:00100000:1.0:1767319094.492105:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2907 00000100:00100000:1.0:1767319094.492112:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985081856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319094.492156:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985081856:12345-0@lo:400:kworker.0 Request processed in 44us (241us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319094.492192:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2907 00000100:00100000:1.0:1767319094.492217:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2908 00000100:00100000:1.0:1767319094.492223:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985081728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319094.492260:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985081728:12345-0@lo:400:kworker.0 Request processed in 36us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319094.492266:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2908 00000100:00100000:1.0:1767319094.492283:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985080960:0@lo:38:kworker.0 00000100:00100000:1.0:1767319094.492308:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319094.492328:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985081088:0@lo:38:kworker.0 00000100:00100000:1.0:1767319094.492348:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319094.492369:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985081344:0@lo:38:kworker.0 00000100:00100000:1.0:1767319094.492387:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319094.492401:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985081600:0@lo:38:kworker.0 00000100:00100000:1.0:1767319094.492419:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319094.492444:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985080960 00000100:00100000:1.0:1767319094.492448:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985080960 00000100:00100000:1.0:1767319094.492461:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081344 00000100:00100000:1.0:1767319094.492463:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081344 00000100:00100000:1.0:1767319094.492472:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081600 00000100:00100000:1.0:1767319094.492474:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081600 00000100:00100000:1.0:1767319094.492484:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4820 00000100:00100000:1.0:1767319094.492488:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985080960:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319094.492505:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319094.492534:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985080960:12345-0@lo:38:kworker.0 Request processed in 45us (225us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319094.492541:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4820 00000100:00100000:1.0:1767319094.492550:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4821 00000100:00100000:1.0:1767319094.492554:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985081344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319094.492562:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319094.492579:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985081344:12345-0@lo:38:kworker.0 Request processed in 24us (191us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319094.492584:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4821 00000100:00100000:1.0:1767319094.492592:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4822 00000100:00100000:1.0:1767319094.492596:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985081600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319094.492604:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319094.492619:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985081600:12345-0@lo:38:kworker.0 Request processed in 22us (200us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319094.492624:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4822 00000100:00100000:1.0:1767319094.492641:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081088 00000100:00100000:1.0:1767319094.492644:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081088 00000100:00100000:1.0:1767319094.492660:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1695 00000100:00100000:1.0:1767319094.492664:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985081088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319094.492672:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319094.492692:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985081088:12345-0@lo:38:kworker.0 Request processed in 27us (343us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319094.492697:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1695 00000100:00100000:1.0:1767319094.492729:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985081728:0@lo:400:kworker.0 00000100:00100000:1.0:1767319094.492739:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985081856:0@lo:400:kworker.0 00000100:00080000:1.0:1767319094.492761:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319094.492765:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319094.492768:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985080960:0@lo:38:kworker.0 00000100:00080000:1.0:1767319094.492774:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319094.492777:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319094.492780:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985081088:0@lo:38:kworker.0 00000100:00080000:1.0:1767319094.492786:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319094.492789:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319094.492791:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985081344:0@lo:38:kworker.0 00000100:00080000:1.0:1767319094.492798:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319094.492800:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319094.492803:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985081600:0@lo:38:kworker.0 00000100:00100000:3.0:1767319094.493372:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985081472:0@lo:400:kworker.0 00000100:00100000:3.0:1767319094.493405:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319094.493434:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319094.493440:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985081216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319094.493462:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319094.493485:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081472 00000100:00100000:3.0:1767319094.493489:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081472 00000100:00100000:3.0:1767319094.493509:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081216 00000100:00100000:3.0:1767319094.493512:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081216 00000100:00100000:3.0:1767319094.493526:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4413 00000100:00100000:3.0:1767319094.493532:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985081472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319094.493565:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985081472:12345-0@lo:400:kworker.0 Request processed in 33us (159us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319094.493572:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4413 00000100:00100000:3.0:1767319094.493584:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4414 00000100:00100000:3.0:1767319094.493590:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985081216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319094.493612:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985081216:12345-0@lo:400:kworker.0 Request processed in 23us (149us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319094.493618:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4414 00000100:00100000:3.0:1767319094.493641:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985081216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319094.493649:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985081472:0@lo:400:kworker.0 00000100:00080000:2.0:1767319094.493725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319094.493730:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319094.493732:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319095.004188:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319095.004215:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985081984:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319095.004288:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319095.004383:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985081984 00000100:00100000:2.0:1767319095.004391:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985081984 00000100:00100000:2.0:1767319095.004445:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2070 00000100:00100000:2.0:1767319095.004454:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985081984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319095.004492:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319095.004535:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985081984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (246us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319095.004542:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2070 00000100:00100000:3.0:1767319095.004810:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985082112:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319095.004865:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319095.004919:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985081984:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319095.006675:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985082112 00000100:00100000:2.0:1767319095.006687:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985082112 00000100:00100000:2.0:1767319095.006753:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2071 00000100:00100000:2.0:1767319095.006764:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985082112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319095.006792:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319095.006861:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985082112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (1995us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319095.006869:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2071 00000100:00100000:0.0:1767319095.006959:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985082112:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319095.131560:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319096.155354:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319097.179409:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319098.203410:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319098.538655:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319098.538707:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319098.538815:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541568 00000100:00100000:0.0:1767319098.538826:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541568 00000100:00100000:0.0:1767319098.538874:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541696 00000100:00100000:0.0:1767319098.538877:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541696 00000100:00100000:0.0:1767319098.538919:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4415 00000100:00080000:3.0:1767319098.538928:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319098.538930:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920541696:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319098.538983:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4823 00000100:00100000:3.0:1767319098.538991:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920541568:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319098.539011:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319098.539020:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920541696:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (311us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319098.539027:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4415 00000100:00100000:3.0:1767319098.539047:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920541568:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (402us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319098.539055:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4823 00010000:00010000:1.0:1767319099.227441:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319099.611279:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319099.611290:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319099.611294:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319099.611298:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319099.611303:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319099.611313:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4342 00000100:00080000:2.0:1767319099.611319:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319099.611325:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319099.611367:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdad80 x1853163985082240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319099.611399:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319099.611404:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319099.611406:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319099.611410:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4342 00000100:00080000:2.0:1767319099.611413:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319099.611417:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319099.611431:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8380 x1853163985082368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319099.611480:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319099.611493:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319099.611495:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319099.611498:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4342 00000100:00080000:2.0:1767319099.611501:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319099.611504:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319099.611519:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9500 x1853163985082624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319099.611536:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319099.611538:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319099.611540:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319099.611543:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4342 00000100:00080000:2.0:1767319099.611546:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319099.611549:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319099.611560:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9180 x1853163985082880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319099.611634:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985082496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319099.611661:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985082240:0@lo:38:kworker.0 00000100:00100000:3.0:1767319099.611706:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319099.611787:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319099.611794:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985082752:0@lo:400:kworker.0 00000100:00100000:3.0:1767319099.611816:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319099.611889:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985082496 00000100:00100000:1.0:1767319099.611896:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985082496 00000100:00100000:1.0:1767319099.611985:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985082752 00000100:00100000:1.0:1767319099.611989:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985082752 00000100:00100000:1.0:1767319099.612011:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4416 00000100:00100000:1.0:1767319099.612019:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985082496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319099.612087:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985082496:12345-0@lo:400:kworker.0 Request processed in 68us (382us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319099.612096:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4416 00000100:00100000:1.0:1767319099.612113:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4417 00000100:00100000:1.0:1767319099.612119:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985082752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319099.612146:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985082752:12345-0@lo:400:kworker.0 Request processed in 26us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319099.612152:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4417 00000100:00100000:1.0:1767319099.612197:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319099.612222:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985082368:0@lo:38:kworker.0 00000100:00100000:1.0:1767319099.612261:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319099.612284:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985082624:0@lo:38:kworker.0 00000100:00100000:1.0:1767319099.612306:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319099.612319:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985082880:0@lo:38:kworker.0 00000100:00100000:1.0:1767319099.612339:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319099.612369:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985082368 00000100:00100000:1.0:1767319099.612373:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985082368 00000100:00100000:1.0:1767319099.612393:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1696 00000100:00100000:1.0:1767319099.612398:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985082368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319099.612418:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319099.612450:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985082368:12345-0@lo:38:kworker.0 Request processed in 51us (188us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319099.612457:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1696 00000100:00100000:1.0:1767319099.612474:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985082240 00000100:00100000:1.0:1767319099.612477:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985082240 00000100:00100000:1.0:1767319099.612494:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985082624 00000100:00100000:1.0:1767319099.612496:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985082624 00000100:00100000:1.0:1767319099.612502:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985082880 00000100:00100000:1.0:1767319099.612505:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985082880 00000100:00100000:1.0:1767319099.612513:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4824 00000100:00100000:1.0:1767319099.612517:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985082240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319099.612528:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319099.612553:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985082240:12345-0@lo:38:kworker.0 Request processed in 34us (355us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319099.612559:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4824 00000100:00100000:1.0:1767319099.612569:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4825 00000100:00100000:1.0:1767319099.612573:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985082624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319099.612582:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319099.612602:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985082624:12345-0@lo:38:kworker.0 Request processed in 28us (296us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319099.612608:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4825 00000100:00100000:1.0:1767319099.612618:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4826 00000100:00100000:1.0:1767319099.612622:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985082880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319099.612631:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319099.612650:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985082880:12345-0@lo:38:kworker.0 Request processed in 26us (310us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319099.612656:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4826 00000100:00100000:1.0:1767319099.612703:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985082752:0@lo:400:kworker.0 00000100:00100000:1.0:1767319099.612715:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985082496:0@lo:400:kworker.0 00000100:00080000:1.0:1767319099.612747:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319099.612752:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319099.612755:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985082240:0@lo:38:kworker.0 00000100:00080000:1.0:1767319099.612762:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319099.612766:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319099.612769:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985082368:0@lo:38:kworker.0 00000100:00080000:1.0:1767319099.612777:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319099.612780:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319099.612783:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985082624:0@lo:38:kworker.0 00000100:00080000:1.0:1767319099.612790:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319099.612793:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319099.612796:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985082880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319099.613253:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985083136:0@lo:400:kworker.0 00000100:00100000:0.0:1767319099.613302:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319099.613342:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319099.613349:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985083008:0@lo:400:kworker.0 00000100:00080000:3.0:1767319099.613368:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319099.613375:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319099.613409:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083136 00000100:00100000:0.0:1767319099.613415:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083136 00000100:00100000:0.0:1767319099.613464:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083008 00000100:00100000:0.0:1767319099.613468:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083008 00000100:00100000:0.0:1767319099.613489:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2909 00000100:00100000:0.0:1767319099.613496:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985083136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319099.613548:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985083136:12345-0@lo:400:kworker.0 Request processed in 51us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319099.613555:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2909 00000100:00100000:0.0:1767319099.613570:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2910 00000100:00100000:0.0:1767319099.613576:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985083008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319099.613600:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985083008:12345-0@lo:400:kworker.0 Request processed in 25us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319099.613605:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2910 00000100:00100000:0.0:1767319099.613637:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985083008:0@lo:400:kworker.0 00000100:00100000:0.0:1767319099.613646:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985083136:0@lo:400:kworker.0 00000100:00080000:3.0:1767319099.614096:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319099.614100:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319100.123606:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319100.123632:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985083264:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319100.123699:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319100.123746:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985083392:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319100.123770:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319100.123860:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083264 00000100:00100000:1.0:1767319100.123869:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083264 00000100:00100000:1.0:1767319100.123923:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083392 00000100:00100000:1.0:1767319100.123926:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083392 00000100:00100000:1.0:1767319100.127040:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2072 00000100:00100000:1.0:1767319100.127064:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985083264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319100.127104:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319100.127156:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2073 00000100:00100000:2.0:1767319100.127183:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985083392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319100.127204:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319100.127241:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985083392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (3470us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319100.127248:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2073 00000100:00100000:1.0:1767319100.127265:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985083264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 211us (3563us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319100.127273:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2072 00000100:00100000:2.0:1767319100.127327:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985083392:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319100.127343:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985083264:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319100.252902:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319101.286989:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319102.300417:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319103.323927:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319103.660283:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319103.660439:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541824 00000100:00100000:2.0:1767319103.660452:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541824 00000100:00100000:2.0:1767319103.660511:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4827 00000100:00100000:2.0:1767319103.660520:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920541824:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319103.660547:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319103.660634:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920541824:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 114us (360us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319103.660642:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4827 00000100:00100000:3.0:1767319103.664260:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319103.664717:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920541952 00000100:00100000:0.0:1767319103.664729:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920541952 00000100:00100000:0.0:1767319103.664805:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4418 00000100:00100000:0.0:1767319103.664816:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920541952:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319103.664924:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920541952:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 109us (673us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319103.664933:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4418 00000100:00080000:3.0:1767319103.666647:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319104.354751:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319104.733350:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319104.733369:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319104.733372:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319104.733376:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319104.733381:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319104.733390:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4347 00000100:00080000:2.0:1767319104.733396:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319104.733402:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319104.733449:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdaa00 x1853163985083520/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319104.733489:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319104.733493:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319104.733495:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319104.733498:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4347 00000100:00080000:2.0:1767319104.733501:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319104.733504:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319104.733519:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bda680 x1853163985083648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319104.733559:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319104.733563:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319104.733565:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319104.733569:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4347 00000100:00080000:2.0:1767319104.733571:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319104.733575:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319104.733586:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8e00 x1853163985083904/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319104.733600:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319104.733603:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319104.733604:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319104.733607:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4347 00000100:00080000:2.0:1767319104.733610:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319104.733612:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319104.733622:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8000 x1853163985084160/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319104.733921:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985083776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319104.733924:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985084416:0@lo:400:kworker.0 00000100:00100000:0.0:1767319104.733996:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985084032:0@lo:400:kworker.0 00000100:00100000:1.0:1767319104.734059:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985083520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319104.741510:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319104.741594:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319104.741606:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985084288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319104.741729:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319104.741774:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084416 00000100:00100000:2.0:1767319104.741783:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084416 00000100:00100000:2.0:1767319104.741963:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084288 00000100:00100000:2.0:1767319104.741968:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084288 00000100:00100000:2.0:1767319104.741993:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2911 00000100:00100000:2.0:1767319104.742003:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985084416:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319104.742476:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985084416:12345-0@lo:400:kworker.0 Request processed in 472us (970us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319104.742485:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2911 00000100:00100000:2.0:1767319104.742504:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2912 00000100:00100000:2.0:1767319104.742510:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985084288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319104.742539:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985084288:12345-0@lo:400:kworker.0 Request processed in 30us (811us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319104.742546:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2912 00000100:00100000:2.0:1767319104.742598:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985084288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319104.742612:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985084416:0@lo:400:kworker.0 00000100:00100000:0.0:1767319104.742640:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319104.742674:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084032 00000100:00100000:2.0:1767319104.742679:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084032 00000100:00100000:2.0:1767319104.742717:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4419 00000100:00100000:2.0:1767319104.742725:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985084032:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319104.742736:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319104.742742:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319104.742745:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319104.742766:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985084032:12345-0@lo:400:kworker.0 Request processed in 42us (127us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319104.742773:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4419 00000100:00100000:2.0:1767319104.742815:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985084032:0@lo:400:kworker.0 00000100:00100000:3.0:1767319104.743439:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319104.746510:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083776 00000100:00100000:2.0:1767319104.746526:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083776 00000100:00100000:2.0:1767319104.746640:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4420 00000100:00100000:2.0:1767319104.746652:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985083776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319104.746774:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985083776:12345-0@lo:400:kworker.0 Request processed in 124us (3336us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319104.746782:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4420 00000100:00080000:0.0:1767319104.746908:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319104.748898:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985083776:0@lo:400:kworker.0 00000100:00100000:1.0:1767319104.750158:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319104.750246:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985083648:0@lo:38:kworker.0 00000100:00100000:1.0:1767319104.750280:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319104.750995:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083520 00000100:00100000:3.0:1767319104.751004:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083520 00000100:00100000:3.0:1767319104.751066:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4828 00000100:00100000:3.0:1767319104.751074:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985083520:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319104.751099:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319104.751152:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985083520:12345-0@lo:38:kworker.0 Request processed in 78us (1000us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319104.751159:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4828 00000100:00100000:1.0:1767319104.751477:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985083904:0@lo:38:kworker.0 00000100:00100000:1.0:1767319104.751505:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319104.751526:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985084160:0@lo:38:kworker.0 00000100:00100000:1.0:1767319104.751546:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319104.751568:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083904 00000100:00100000:1.0:1767319104.751572:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083904 00000100:00100000:1.0:1767319104.751591:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084160 00000100:00100000:1.0:1767319104.751593:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084160 00000100:00100000:1.0:1767319104.751606:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4829 00000100:00100000:1.0:1767319104.751611:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985083904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319104.751623:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319104.751657:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985083904:12345-0@lo:38:kworker.0 Request processed in 44us (151us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319104.751663:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4829 00000100:00100000:1.0:1767319104.751674:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4830 00000100:00100000:1.0:1767319104.751677:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985084160:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319104.751686:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319104.751702:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985084160:12345-0@lo:38:kworker.0 Request processed in 24us (156us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319104.751708:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4830 00000100:00080000:1.0:1767319104.751740:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319104.751745:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319104.751748:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985084160:0@lo:38:kworker.0 00000100:00080000:1.0:1767319104.751772:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319104.751776:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319104.751779:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985083520:0@lo:38:kworker.0 00000100:00080000:1.0:1767319104.751786:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319104.751789:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319104.751791:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985083904:0@lo:38:kworker.0 00000100:00100000:3.0:1767319104.752218:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985083648 00000100:00100000:3.0:1767319104.752223:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985083648 00000100:00100000:3.0:1767319104.752253:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1697 00000100:00100000:3.0:1767319104.752259:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985083648:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319104.752270:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319104.752318:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985083648:12345-0@lo:38:kworker.0 Request processed in 59us (2037us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319104.752324:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1697 00000100:00080000:0.0:1767319104.752381:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319104.752386:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319104.752389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985083648:0@lo:38:kworker.0 00000100:00100000:0.0:1767319105.245535:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319105.245560:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985084544:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319105.245630:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319105.245679:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084544 00000100:00100000:0.0:1767319105.245687:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084544 00000100:00100000:0.0:1767319105.245731:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2074 00000100:00100000:0.0:1767319105.245740:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985084544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319105.245771:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319105.245813:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985084544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (183us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319105.245821:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2074 00000100:00100000:0.0:1767319105.245904:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985084544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319105.246471:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985084672:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319105.246519:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319105.246817:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084672 00000100:00100000:0.0:1767319105.246823:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084672 00000100:00100000:0.0:1767319105.246873:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2075 00000100:00100000:0.0:1767319105.246881:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985084672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319105.246901:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319105.246947:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985084672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (428us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319105.246954:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2075 00000100:00100000:3.0:1767319105.247039:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985084672:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319105.372870:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319106.398213:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319107.421045:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319108.443526:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319108.778217:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319108.778276:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319108.778384:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920542208 00000100:00100000:2.0:1767319108.778398:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920542208 00000100:00100000:2.0:1767319108.778477:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920542080 00000100:00100000:2.0:1767319108.778481:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920542080 00000100:00100000:2.0:1767319108.778506:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4421 00000100:00100000:2.0:1767319108.778518:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920542208:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319108.778604:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319108.778611:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920542208:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 95us (444us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319108.778620:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4421 00000100:00100000:2.0:1767319108.778640:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4831 00000100:00100000:2.0:1767319108.778644:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920542080:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319108.778665:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319108.778695:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920542080:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (418us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319108.778702:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4831 00010000:00010000:1.0:1767319109.467352:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319109.851351:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319109.851362:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319109.851366:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319109.851371:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319109.851375:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319109.851383:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4352 00000100:00080000:1.0:1767319109.851389:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319109.851396:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319109.851438:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfa680 x1853163985084800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319109.851464:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319109.851468:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319109.851469:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319109.851473:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4352 00000100:00080000:1.0:1767319109.851476:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319109.851479:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319109.851493:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985084928/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319109.851541:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319109.851546:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319109.851547:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319109.851551:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4352 00000100:00080000:1.0:1767319109.851553:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319109.851556:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319109.851578:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985085184/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319109.851601:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319109.851604:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319109.851606:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319109.851609:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4352 00000100:00080000:1.0:1767319109.851611:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319109.851615:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319109.851629:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518380 x1853163985085440/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319109.851717:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985085056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319109.851794:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319109.851820:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985085312:0@lo:400:kworker.0 00000100:00100000:2.0:1767319109.851857:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319109.851882:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985085696:0@lo:400:kworker.0 00000100:00100000:2.0:1767319109.851914:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985085568:0@lo:400:kworker.0 00000100:00100000:3.0:1767319109.851915:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319109.851948:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319109.851960:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085696 00000100:00100000:3.0:1767319109.851968:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085696 00000100:00100000:2.0:1767319109.851985:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085568 00000100:00100000:2.0:1767319109.851991:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085568 00000100:00100000:1.0:1767319109.852056:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085056 00000100:00100000:1.0:1767319109.852061:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085056 00000100:00100000:1.0:1767319109.852092:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085312 00000100:00100000:1.0:1767319109.852095:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085312 00000100:00100000:1.0:1767319109.852118:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4422 00000100:00100000:1.0:1767319109.852126:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985085056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319109.852212:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985085056:12345-0@lo:400:kworker.0 Request processed in 86us (417us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319109.852219:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4422 00000100:00100000:1.0:1767319109.852236:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4423 00000100:00100000:1.0:1767319109.852242:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985085312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319109.852272:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985085312:12345-0@lo:400:kworker.0 Request processed in 31us (415us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319109.852278:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4423 00000100:00100000:1.0:1767319109.852326:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985085312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319109.852399:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2913 00000100:00100000:3.0:1767319109.852407:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985085696:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319109.852443:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985085696:12345-0@lo:400:kworker.0 Request processed in 37us (528us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319109.852451:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2913 00000100:00100000:3.0:1767319109.852464:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2914 00000100:00100000:3.0:1767319109.852469:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985085568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319109.852499:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985085568:12345-0@lo:400:kworker.0 Request processed in 29us (550us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319109.852504:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2914 00000100:00100000:0.0:1767319109.852882:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985084800:0@lo:38:kworker.0 00000100:00100000:0.0:1767319109.852916:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319109.852957:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985084928:0@lo:38:kworker.0 00000100:00100000:0.0:1767319109.852979:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319109.853000:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985085184:0@lo:38:kworker.0 00000100:00100000:0.0:1767319109.853019:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319109.853034:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985085440:0@lo:38:kworker.0 00000100:00100000:0.0:1767319109.853064:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319109.853093:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084928 00000100:00100000:0.0:1767319109.853098:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084928 00000100:00100000:0.0:1767319109.853123:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1698 00000100:00100000:0.0:1767319109.853129:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985084928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319109.853149:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319109.853197:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985084928:12345-0@lo:38:kworker.0 Request processed in 67us (217us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319109.853207:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1698 00000100:00100000:0.0:1767319109.853226:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985084800 00000100:00100000:0.0:1767319109.853229:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985084800 00000100:00100000:0.0:1767319109.853245:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085184 00000100:00100000:0.0:1767319109.853248:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085184 00000100:00100000:0.0:1767319109.853261:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085440 00000100:00100000:0.0:1767319109.853274:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085440 00000100:00100000:0.0:1767319109.853291:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4832 00000100:00100000:0.0:1767319109.853295:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985084800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319109.853305:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319109.853329:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985084800:12345-0@lo:38:kworker.0 Request processed in 32us (413us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319109.853334:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4832 00000100:00100000:0.0:1767319109.853345:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4833 00000100:00100000:0.0:1767319109.853349:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985085184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319109.853357:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319109.853377:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985085184:12345-0@lo:38:kworker.0 Request processed in 28us (358us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319109.853383:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4833 00000100:00100000:0.0:1767319109.853393:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4834 00000100:00100000:0.0:1767319109.853396:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985085440:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319109.853404:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319109.853423:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985085440:12345-0@lo:38:kworker.0 Request processed in 25us (357us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319109.853427:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4834 00000100:00100000:0.0:1767319109.853464:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985085056:0@lo:400:kworker.0 00000100:00080000:0.0:1767319109.853490:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319109.853495:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319109.853498:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985084800:0@lo:38:kworker.0 00000100:00080000:0.0:1767319109.853505:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319109.853508:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319109.853510:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985084928:0@lo:38:kworker.0 00000100:00080000:0.0:1767319109.853517:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319109.853519:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319109.853522:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985085184:0@lo:38:kworker.0 00000100:00080000:0.0:1767319109.853528:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319109.853531:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319109.853534:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985085440:0@lo:38:kworker.0 00000100:00080000:0.0:1767319109.853571:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319109.853577:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319109.853579:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319109.853614:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985085696:0@lo:400:kworker.0 00000100:00100000:0.0:1767319109.853642:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985085568:0@lo:400:kworker.0 00000100:00100000:0.0:1767319110.368355:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985085824:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319110.368427:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319110.368580:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085824 00000100:00100000:1.0:1767319110.368588:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085824 00000100:00100000:1.0:1767319110.368636:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2076 00000100:00100000:1.0:1767319110.368644:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985085824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319110.368673:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319110.368721:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985085824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (295us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319110.368728:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2076 00000100:00100000:3.0:1767319110.368817:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985085824:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319110.369345:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985085952:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319110.369380:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319110.369534:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985085952 00000100:00100000:1.0:1767319110.369541:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985085952 00000100:00100000:1.0:1767319110.369575:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2077 00000100:00100000:1.0:1767319110.369582:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985085952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319110.369602:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319110.369641:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985085952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319110.369649:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2077 00000100:00100000:3.0:1767319110.369711:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985085952:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319110.491546:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319111.515601:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319112.539494:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319113.563388:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319113.898278:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319113.898386:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920542464 00000100:00100000:0.0:1767319113.898396:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920542464 00000100:00100000:0.0:1767319113.898486:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4424 00000100:00100000:0.0:1767319113.898496:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920542464:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319113.898544:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319113.898579:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920542464:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (309us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319113.898588:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4424 00000100:00100000:3.0:1767319113.901255:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319113.901294:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920542592 offset 0 length 520: 4 00000100:00100000:1.0:1767319113.901363:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920542336 00000100:00100000:1.0:1767319113.901377:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920542336 00000100:00100000:1.0:1767319113.901628:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4835 00000100:00100000:1.0:1767319113.901636:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920542336:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319113.901661:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319113.901765:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920542336:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 127us (523us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319113.901772:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4835 00010000:00010000:0.0:1767319114.595790:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319114.971485:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319089/real 1767319089] req@ffff8ce1a4dfb100 x1853163985079552/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319114 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319114.971531:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a4dfb100 x1853163985079552/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319114 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319114.971558:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319114.971577:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319114.971582:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985079552:0@lo:250:kworker.0 00000100:00080000:3.0:1767319114.971921:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319114.971928:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319114.971932:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319114.971937:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4337 00000100:00080000:3.0:1767319114.971943:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319114.971950:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319114.971984:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524df80 x1853163985086080/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319114.972014:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319114.972017:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319114.972020:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319114.972023:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4358 00000100:00080000:3.0:1767319114.972026:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319114.972030:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319114.972044:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985086208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319114.972055:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319114.972059:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319114.972061:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319114.972066:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4358 00000100:00080000:3.0:1767319114.972069:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319114.972072:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319114.972083:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985086080:0@lo:250:kworker.0 00000100:00100000:3.0:1767319114.972089:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f800 x1853163985086336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319114.972131:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319114.972136:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319114.972138:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319114.972142:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4358 00000100:00080000:3.0:1767319114.972145:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319114.972150:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000400:00000100:1.0:1767319114.972156:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985086080 offset 0 length 520: 4 00000100:00100000:3.0:1767319114.972196:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f480 x1853163985086592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319114.972239:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985086208:0@lo:38:kworker.0 00000100:00100000:1.0:1767319114.972274:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319114.972308:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985086336:0@lo:38:kworker.0 00000100:00100000:1.0:1767319114.972333:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319114.972351:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985086592:0@lo:38:kworker.0 00000100:00100000:1.0:1767319114.972372:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319114.972414:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985086208 00000100:00100000:1.0:1767319114.972421:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985086208 00000100:00100000:1.0:1767319114.972454:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985086592 00000100:00100000:1.0:1767319114.972457:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985086592 00000100:00100000:0.0:1767319114.972460:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319114.972477:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985086464:0@lo:400:kworker.0 00000100:00100000:1.0:1767319114.972498:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4836 00000100:00100000:1.0:1767319114.972507:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985086208:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319114.972540:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319114.972587:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985086336 00000100:00100000:0.0:1767319114.972594:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985086336 00000100:00100000:0.0:1767319114.972625:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1699 00000100:00100000:0.0:1767319114.972632:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985086336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319114.972674:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319114.972739:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985086336:12345-0@lo:38:kworker.0 Request processed in 105us (403us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319114.972746:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1699 00000100:00100000:2.0:1767319114.972768:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985086464 00000100:00100000:2.0:1767319114.972776:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985086464 00000100:00100000:0.0:1767319114.972801:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4837 00000100:00100000:0.0:1767319114.972812:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985086592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319114.972823:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319114.972836:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4425 00000100:00100000:2.0:1767319114.972851:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985086464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319114.972858:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985086592:12345-0@lo:38:kworker.0 Request processed in 47us (483us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319114.972868:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4837 00000100:00100000:2.0:1767319114.972925:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985086464:12345-0@lo:400:kworker.0 Request processed in 78us (383us total) trans 0 rc 0/0 00000100:00080000:0.0:1767319114.972930:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767319114.972938:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4425 00000100:00080000:0.0:1767319114.972942:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319114.972948:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985086336:0@lo:38:kworker.0 00000100:00080000:0.0:1767319114.972966:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319114.972979:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319114.973016:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985086464:0@lo:400:kworker.0 00010000:00020000:1.0:1767319114.973871:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319114.973920:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985086208:12345-0@lo:38:kworker.0 Request processed in 1412us (1645us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319114.973928:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4836 00000100:00080000:0.0:1767319114.974082:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319114.974088:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985086592:0@lo:38:kworker.0 00000100:00080000:0.0:1767319114.974115:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319114.974119:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319114.974121:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985086208:0@lo:38:kworker.0 00000100:00080000:3.0:1767319114.974375:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319114.974381:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319114.974387:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319114.974393:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4358 00000100:00080000:3.0:1767319114.974397:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319114.974403:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319114.974411:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985086720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319114.974439:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985086848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319114.974462:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319114.974680:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985086976:0@lo:400:kworker.0 00000100:00100000:0.0:1767319114.974778:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985087104:0@lo:400:kworker.0 00000100:00100000:1.0:1767319114.975274:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985086720 00000100:00100000:1.0:1767319114.975288:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985086720 00000100:00100000:1.0:1767319114.975748:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4426 00000100:00100000:1.0:1767319114.975760:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985086720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319114.977063:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319114.977741:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319114.977798:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985087104 00000100:00080000:0.0:1767319114.977809:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319114.977809:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985087104 00000100:00100000:0.0:1767319114.977832:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985086848:0@lo:38:kworker.0 00000100:00100000:3.0:1767319114.977879:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985086976 00000100:00100000:3.0:1767319114.977882:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985086976 00000100:00100000:0.0:1767319114.977886:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319114.977911:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2915 00000100:00100000:3.0:1767319114.977920:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985087104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319114.977928:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985086848 00000100:00100000:0.0:1767319114.977931:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985086848 00000100:00100000:0.0:1767319114.977955:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4838 00000100:00100000:0.0:1767319114.977960:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985086848:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319114.977977:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985087104:12345-0@lo:400:kworker.0 Request processed in 60us (920us total) trans 0 rc 0/0 00010000:00020000:0.0:1767319114.977983:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319114.977984:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2915 00000100:00100000:0.0:1767319114.978016:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985086848:12345-0@lo:38:kworker.0 Request processed in 55us (129us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319114.978025:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4838 00000100:00080000:0.0:1767319114.978075:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319114.978081:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319114.978085:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985086848:0@lo:38:kworker.0 00000100:00100000:3.0:1767319114.978454:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2916 00000100:00100000:3.0:1767319114.978467:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985086976:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319114.978512:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985086976:12345-0@lo:400:kworker.0 Request processed in 49us (772us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319114.978611:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2916 00000100:00100000:3.0:1767319114.978646:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985086976:0@lo:400:kworker.0 00000100:00080000:0.0:1767319114.978655:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319114.978659:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319114.978688:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985087104:0@lo:400:kworker.0 00000100:00100000:1.0:1767319114.979480:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985086720:12345-0@lo:400:kworker.0 Request processed in 3718us (5017us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319114.979491:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4426 00000100:00100000:2.0:1767319114.980976:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985086720:0@lo:400:kworker.0 00000100:00100000:2.0:1767319115.483422:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319115.483454:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985087232:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319115.483560:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319115.484146:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319115.484159:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985087360:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319115.484198:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985087232 00000100:00100000:3.0:1767319115.484209:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985087232 00000100:00100000:0.0:1767319115.484265:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319115.484270:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2078 00000100:00100000:3.0:1767319115.484283:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985087232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319115.484325:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319115.484377:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985087232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (817us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319115.484385:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2078 00000100:00100000:3.0:1767319115.484404:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985087360 00000100:00100000:3.0:1767319115.484408:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985087360 00000100:00100000:2.0:1767319115.484539:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985087232:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319115.485705:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2079 00000100:00100000:3.0:1767319115.485723:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985087360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319115.485750:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319115.485813:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985087360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (1548us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319115.485821:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2079 00000100:00100000:1.0:1767319115.489007:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985087360:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319115.611491:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319116.636226:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319117.659591:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319118.683438:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319119.017915:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319119.017991:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319119.018055:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920542848 00000100:00100000:1.0:1767319119.018071:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920542848 00000100:00100000:1.0:1767319119.018231:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920542720 00000100:00100000:1.0:1767319119.018236:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920542720 00000100:00100000:3.0:1767319119.018248:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4427 00000100:00100000:3.0:1767319119.018267:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920542848:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319119.018367:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920542848:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 103us (457us total) trans 0 rc 0/0 00000100:00080000:2.0:1767319119.018376:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319119.018446:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4839 00000100:00100000:1.0:1767319119.018456:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920542720:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319119.018485:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319119.018665:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920542720:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 207us (671us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319119.018676:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4839 00000100:00100000:3.0:1767319119.027560:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4427 00010000:00010000:2.0:1767319119.708053:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319120.091473:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319120.091486:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319120.091490:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319120.091495:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319120.091499:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319120.091508:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4363 00000100:00080000:3.0:1767319120.091514:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319120.091522:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319120.091596:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f100 x1853163985087488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319120.091682:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985087488:0@lo:38:kworker.0 00000100:00100000:1.0:1767319120.091772:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319120.091959:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985087488 00000100:00100000:2.0:1767319120.091971:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985087488 00000100:00100000:2.0:1767319120.092023:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4840 00000100:00100000:2.0:1767319120.092033:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985087488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319120.092058:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319120.092235:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319120.092240:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319120.092242:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319120.092247:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4363 00000100:00080000:3.0:1767319120.092250:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319120.092255:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319120.092274:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524dc00 x1853163985087616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319120.092678:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985087616:0@lo:38:kworker.0 00000100:00100000:1.0:1767319120.092721:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319120.092799:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985087616 00000100:00100000:0.0:1767319120.092805:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985087616 00000100:00100000:0.0:1767319120.092834:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1700 00000100:00100000:0.0:1767319120.092840:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985087616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319120.092855:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319120.092887:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985087616:12345-0@lo:38:kworker.0 Request processed in 46us (165us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319120.092895:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1700 00000100:00100000:2.0:1767319120.093259:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985087488:12345-0@lo:38:kworker.0 Request processed in 1224us (1485us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319120.093268:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4840 00000100:00080000:1.0:1767319120.093347:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319120.093353:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319120.093357:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985087488:0@lo:38:kworker.0 00000100:00080000:1.0:1767319120.093382:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319120.093386:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319120.093389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985087616:0@lo:38:kworker.0 00000100:00100000:0.0:1767319120.093561:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985087744:0@lo:400:kworker.0 00000100:00100000:0.0:1767319120.093610:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319120.093668:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985087744 00000100:00100000:2.0:1767319120.093674:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985087744 00000100:00100000:2.0:1767319120.093730:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4428 00000100:00100000:2.0:1767319120.093739:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985087744:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319120.093799:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319120.093805:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319120.093807:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319120.093811:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4363 00000100:00080000:3.0:1767319120.093814:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319120.093818:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319120.093839:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524e300 x1853163985087872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319120.093896:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319120.093910:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985087872:0@lo:38:kworker.0 00000100:00100000:1.0:1767319120.093953:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319120.093986:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985087872 00000100:00100000:1.0:1767319120.093991:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985087872 00000100:00100000:1.0:1767319120.094014:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4841 00000100:00100000:1.0:1767319120.094019:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985087872:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319120.094035:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319120.094068:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985087872:12345-0@lo:38:kworker.0 Request processed in 48us (114us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319120.094075:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4841 00000100:00080000:1.0:1767319120.094100:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319120.094103:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319120.094106:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985087872:0@lo:38:kworker.0 00000100:00100000:1.0:1767319120.094139:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985088000:0@lo:400:kworker.0 00000100:00100000:1.0:1767319120.094180:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319120.094205:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088000 00000100:00100000:1.0:1767319120.094208:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088000 00000100:00100000:1.0:1767319120.094237:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4429 00000100:00100000:1.0:1767319120.094243:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985088000:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319120.094276:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985088000:12345-0@lo:400:kworker.0 Request processed in 33us (112us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319120.094282:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4429 00000100:00080000:1.0:1767319120.094299:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319120.094316:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985088000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319120.094588:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985087744:12345-0@lo:400:kworker.0 Request processed in 850us (976us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319120.094596:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4428 00000100:00080000:3.0:1767319120.094664:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319120.094669:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319120.094672:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319120.094676:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4363 00000100:00080000:3.0:1767319120.094679:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319120.094683:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319120.094703:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d500 x1853163985088128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319120.094809:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985088256:0@lo:400:kworker.0 00000100:00100000:2.0:1767319120.094848:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319120.094879:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319120.094885:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985088384:0@lo:400:kworker.0 00000100:00100000:2.0:1767319120.094908:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319120.094956:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088256 00000100:00100000:2.0:1767319120.094961:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088256 00000100:00100000:2.0:1767319120.094997:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088384 00000100:00100000:2.0:1767319120.095000:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088384 00000100:00100000:2.0:1767319120.095022:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2917 00000100:00100000:2.0:1767319120.095028:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985088256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319120.095063:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985088256:12345-0@lo:400:kworker.0 Request processed in 35us (215us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319120.095069:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2917 00000100:00100000:2.0:1767319120.095080:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2918 00000100:00100000:2.0:1767319120.095085:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985088384:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319120.095111:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985088384:12345-0@lo:400:kworker.0 Request processed in 26us (202us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319120.095116:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2918 00000100:00100000:2.0:1767319120.095146:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985088384:0@lo:400:kworker.0 00000100:00100000:2.0:1767319120.095156:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985088256:0@lo:400:kworker.0 00000100:00080000:2.0:1767319120.095196:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319120.095199:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319120.095436:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985088128:0@lo:38:kworker.0 00000100:00100000:1.0:1767319120.095469:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319120.096072:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088128 00000100:00100000:2.0:1767319120.096079:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088128 00000100:00100000:2.0:1767319120.096119:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4842 00000100:00100000:2.0:1767319120.096126:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985088128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319120.096146:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319120.096541:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985088128:12345-0@lo:38:kworker.0 Request processed in 413us (1069us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319120.096553:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4842 00000100:00080000:1.0:1767319120.096645:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319120.096650:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319120.096654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985088128:0@lo:38:kworker.0 00000100:00100000:0.0:1767319120.096746:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985087744:0@lo:400:kworker.0 00000100:00100000:0.0:1767319120.605378:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985088512:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319120.605494:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319120.605548:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985088640:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319120.605593:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088512 00000100:00100000:0.0:1767319120.605604:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088512 00000100:00100000:2.0:1767319120.605615:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319120.605667:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088640 00000100:00100000:0.0:1767319120.605671:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088640 00000100:00100000:2.0:1767319120.605675:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2080 00000100:00100000:2.0:1767319120.605689:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985088512:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319120.605702:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2081 00000100:00100000:0.0:1767319120.605713:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985088640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319120.605733:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:0.0:1767319120.605735:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319120.605794:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985088512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319120.605800:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985088640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (183us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319120.605809:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2080 00000100:00100000:3.0:1767319120.606026:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985088512:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319120.606070:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985088640:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319120.616991:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2081 00010000:00010000:0.0:1767319120.732348:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319121.755401:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319122.780980:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319123.803521:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319124.142544:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319124.142645:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319124.148890:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920542976 00000100:00100000:3.0:1767319124.148912:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920542976 00000100:00100000:3.0:1767319124.148967:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920543104 00000100:00100000:3.0:1767319124.148971:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920543104 00000100:00100000:3.0:1767319124.149030:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4430 00000100:00100000:3.0:1767319124.149041:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920543104:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319124.149124:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920543104:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (6479us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319124.149132:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4430 00000100:00100000:3.0:1767319124.149153:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4843 00000100:00100000:3.0:1767319124.149159:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920542976:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319124.149197:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319124.149222:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920542976:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 62us (6690us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319124.149589:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319124.149799:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4843 00010000:00010000:2.0:1767319124.827692:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319125.211296:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319125.211314:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319125.211318:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319125.211324:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319125.211328:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319125.211336:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4368 00000100:00080000:3.0:1767319125.211340:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319125.211346:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319125.211388:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f100 x1853163985088768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319125.211416:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319125.211419:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319125.211421:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319125.211425:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4368 00000100:00080000:3.0:1767319125.211427:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319125.211432:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319125.211451:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985088768:0@lo:38:kworker.0 00000100:00100000:3.0:1767319125.211454:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c380 x1853163985088896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319125.211534:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319125.211542:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767319125.211543:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319125.211551:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319125.211556:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4368 00000100:00080000:3.0:1767319125.211558:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319125.211562:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319125.211575:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985089152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319125.211592:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319125.211596:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319125.211598:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319125.211601:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4368 00000100:00080000:3.0:1767319125.211604:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319125.211607:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319125.211618:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985088896:0@lo:38:kworker.0 00000100:00100000:3.0:1767319125.211620:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f480 x1853163985089408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319125.211656:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319125.211680:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985089664:0@lo:400:kworker.0 00000100:00100000:1.0:1767319125.211688:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985089152:0@lo:38:kworker.0 00000100:00100000:3.0:1767319125.211724:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319125.211725:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319125.211803:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985089280:0@lo:400:kworker.0 00000100:00100000:2.0:1767319125.211807:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985089024:0@lo:400:kworker.0 00000100:00100000:1.0:1767319125.215400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985089408:0@lo:38:kworker.0 00000100:00100000:1.0:1767319125.215487:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319125.215553:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088768 00000100:00100000:1.0:1767319125.215561:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088768 00000100:00100000:1.0:1767319125.215594:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089152 00000100:00100000:1.0:1767319125.215597:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089152 00000100:00100000:3.0:1767319125.215644:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319125.215655:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985089536:0@lo:400:kworker.0 00000100:00100000:3.0:1767319125.215688:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319125.215717:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089664 00000100:00100000:3.0:1767319125.215722:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089664 00000100:00100000:3.0:1767319125.216562:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089536 00000100:00100000:3.0:1767319125.216570:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089536 00000100:00100000:3.0:1767319125.216605:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2919 00000100:00100000:3.0:1767319125.216614:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985089664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319125.216721:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985089664:12345-0@lo:400:kworker.0 Request processed in 108us (4999us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319125.216729:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2919 00000100:00100000:3.0:1767319125.216744:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2920 00000100:00100000:3.0:1767319125.216750:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985089536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319125.216876:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985089536:12345-0@lo:400:kworker.0 Request processed in 126us (1187us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319125.216883:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2920 00000100:00100000:3.0:1767319125.216953:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985089536:0@lo:400:kworker.0 00000100:00100000:3.0:1767319125.216967:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985089664:0@lo:400:kworker.0 00000100:00100000:0.0:1767319125.217032:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319125.217077:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985088896 00000100:00100000:0.0:1767319125.217082:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985088896 00000100:00100000:0.0:1767319125.217106:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1701 00000100:00100000:0.0:1767319125.217112:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985088896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319125.217131:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319125.217184:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985088896:12345-0@lo:38:kworker.0 Request processed in 70us (5528us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319125.217191:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1701 00000100:00100000:0.0:1767319125.217212:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089408 00000100:00100000:0.0:1767319125.217215:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089408 00000100:00100000:0.0:1767319125.217228:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089280 00000100:00100000:0.0:1767319125.217230:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089280 00000100:00100000:0.0:1767319125.217248:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4431 00000100:00100000:0.0:1767319125.217255:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985089280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319125.217290:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985089280:12345-0@lo:400:kworker.0 Request processed in 35us (259us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319125.217297:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4431 00000100:00100000:0.0:1767319125.217309:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4844 00000100:00100000:0.0:1767319125.217313:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985088768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319125.217323:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319125.217346:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985088768:12345-0@lo:38:kworker.0 Request processed in 32us (5809us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319125.217352:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4844 00000100:00100000:0.0:1767319125.217363:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4845 00000100:00100000:0.0:1767319125.217367:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985089152:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319125.217375:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319125.217398:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985089152:12345-0@lo:38:kworker.0 Request processed in 30us (5676us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319125.217403:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4845 00000100:00100000:0.0:1767319125.217414:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4846 00000100:00100000:0.0:1767319125.217417:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985089408:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319125.217426:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319125.217447:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985089408:12345-0@lo:38:kworker.0 Request processed in 29us (1961us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319125.217453:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4846 00000100:00080000:3.0:1767319125.217715:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319125.217720:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319125.217723:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319125.217875:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319125.217899:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089024 00000100:00100000:2.0:1767319125.217904:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089024 00000100:00100000:2.0:1767319125.217931:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4432 00000100:00100000:2.0:1767319125.217937:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985089024:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319125.217966:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985089024:12345-0@lo:400:kworker.0 Request processed in 29us (91us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319125.217972:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4432 00000100:00080000:2.0:1767319125.217985:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319125.218000:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985089024:0@lo:400:kworker.0 00000100:00100000:2.0:1767319125.218053:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985089280:0@lo:400:kworker.0 00000100:00080000:3.0:1767319125.218102:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319125.218108:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319125.218111:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985089408:0@lo:38:kworker.0 00000100:00080000:3.0:1767319125.218122:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319125.218125:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319125.218128:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985088896:0@lo:38:kworker.0 00000100:00080000:3.0:1767319125.218135:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319125.218138:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319125.218141:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985089152:0@lo:38:kworker.0 00000100:00080000:3.0:1767319125.218148:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319125.218151:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319125.218154:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985088768:0@lo:38:kworker.0 00000100:00100000:2.0:1767319125.725437:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319125.725499:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985089792:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319125.725576:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319125.725687:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089792 00000100:00100000:0.0:1767319125.725695:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089792 00000100:00100000:0.0:1767319125.725749:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2082 00000100:00100000:0.0:1767319125.725759:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985089792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319125.725789:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319125.725840:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985089792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (264us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319125.725848:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2082 00000100:00100000:2.0:1767319125.725969:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985089792:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319125.726996:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319125.727010:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985089920:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319125.727071:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319125.727190:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985089920 00000100:00100000:1.0:1767319125.727198:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985089920 00000100:00100000:1.0:1767319125.727253:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2083 00000100:00100000:1.0:1767319125.727262:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985089920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319125.727291:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319125.727341:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985089920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (269us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319125.727348:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2083 00000100:00100000:3.0:1767319125.727480:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985089920:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319125.851391:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319126.876257:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319127.899845:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319128.924475:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319129.259276:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319129.259316:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319129.259428:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920543232 00000100:00100000:2.0:1767319129.259444:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920543232 00000100:00100000:2.0:1767319129.259484:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920543360 00000100:00100000:2.0:1767319129.259487:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920543360 00000100:00100000:2.0:1767319129.259530:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4433 00000100:00100000:2.0:1767319129.259538:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920543360:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319129.259609:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920543360:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (291us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319129.259616:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4433 00000100:00100000:2.0:1767319129.259633:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4847 00000100:00100000:2.0:1767319129.259637:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920543232:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319129.259656:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319129.259679:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920543232:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (414us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319129.259684:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4847 00000100:00080000:3.0:1767319129.260125:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319129.954042:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319130.331688:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319130.331698:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319130.331702:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319130.331706:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319130.331709:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319130.331718:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4373 00000100:00080000:0.0:1767319130.331723:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319130.331729:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319130.331762:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0a80 x1853163985090048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319130.331790:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319130.331794:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319130.331796:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319130.331799:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4373 00000100:00080000:0.0:1767319130.331802:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319130.331806:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319130.331821:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1f80 x1853163985090176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319130.331867:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319130.331871:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319130.331874:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319130.331877:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4373 00000100:00080000:0.0:1767319130.331880:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319130.331883:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319130.331908:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adf100 x1853163985090432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319130.331923:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319130.331926:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319130.331928:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319130.331932:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4373 00000100:00080000:0.0:1767319130.331934:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319130.331938:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319130.331946:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adfb80 x1853163985090688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319130.332107:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6aded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985090560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332218:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319130.332274:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319130.332281:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985090304:0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332304:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319130.332331:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6add880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985090944:0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332358:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319130.332413:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319130.332419:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985090816:0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332440:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319130.332468:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090560 00000100:00100000:2.0:1767319130.332475:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090560 00000100:00100000:2.0:1767319130.332519:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090304 00000100:00100000:2.0:1767319130.332522:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090304 00000100:00100000:2.0:1767319130.332544:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4434 00000100:00100000:2.0:1767319130.332552:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985090560:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332599:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985090560:12345-0@lo:400:kworker.0 Request processed in 47us (390us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319130.332606:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4434 00000100:00100000:2.0:1767319130.332622:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4435 00000100:00100000:2.0:1767319130.332628:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985090304:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332652:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985090304:12345-0@lo:400:kworker.0 Request processed in 25us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319130.332658:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4435 00000100:00100000:2.0:1767319130.332674:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090944 00000100:00100000:2.0:1767319130.332677:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090944 00000100:00100000:2.0:1767319130.332700:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090816 00000100:00100000:2.0:1767319130.332703:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090816 00000100:00100000:2.0:1767319130.332722:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2921 00000100:00100000:2.0:1767319130.332729:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985090944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332758:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985090944:12345-0@lo:400:kworker.0 Request processed in 30us (400us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319130.332764:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2921 00000100:00100000:2.0:1767319130.332776:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2922 00000100:00100000:2.0:1767319130.332782:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985090816:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332806:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985090816:12345-0@lo:400:kworker.0 Request processed in 25us (365us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319130.332811:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2922 00000100:00080000:2.0:1767319130.332829:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319130.332833:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319130.332835:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319130.332871:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985090816:0@lo:400:kworker.0 00000100:00100000:2.0:1767319130.332880:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6add880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985090944:0@lo:400:kworker.0 00000100:00100000:3.0:1767319130.333370:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985090048:0@lo:38:kworker.0 00000100:00100000:3.0:1767319130.333422:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319130.333448:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985090176:0@lo:38:kworker.0 00000100:00100000:3.0:1767319130.333469:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319130.333492:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985090432:0@lo:38:kworker.0 00000100:00100000:3.0:1767319130.333510:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319130.333524:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985090688:0@lo:38:kworker.0 00000100:00100000:3.0:1767319130.333539:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319130.333571:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090176 00000100:00100000:3.0:1767319130.333575:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090176 00000100:00100000:3.0:1767319130.333596:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1702 00000100:00100000:3.0:1767319130.333601:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985090176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319130.333619:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319130.333653:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985090176:12345-0@lo:38:kworker.0 Request processed in 50us (183us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319130.333658:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1702 00000100:00100000:3.0:1767319130.333677:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090048 00000100:00100000:3.0:1767319130.333680:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090048 00000100:00100000:3.0:1767319130.333695:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090432 00000100:00100000:3.0:1767319130.333697:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090432 00000100:00100000:3.0:1767319130.333704:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985090688 00000100:00100000:3.0:1767319130.333706:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985090688 00000100:00100000:3.0:1767319130.333715:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4848 00000100:00100000:3.0:1767319130.333719:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985090048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319130.333727:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319130.333751:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985090048:12345-0@lo:38:kworker.0 Request processed in 31us (329us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319130.333757:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4848 00000100:00100000:3.0:1767319130.333766:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4849 00000100:00100000:3.0:1767319130.333770:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985090432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319130.333778:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319130.333797:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985090432:12345-0@lo:38:kworker.0 Request processed in 25us (285us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319130.333803:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4849 00000100:00100000:3.0:1767319130.333812:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4850 00000100:00100000:3.0:1767319130.333815:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985090688:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319130.333823:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319130.333843:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985090688:12345-0@lo:38:kworker.0 Request processed in 26us (302us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319130.333848:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4850 00000100:00080000:3.0:1767319130.333878:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319130.333883:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319130.333886:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985090048:0@lo:38:kworker.0 00000100:00080000:3.0:1767319130.334133:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319130.334138:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319130.334141:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985090176:0@lo:38:kworker.0 00000100:00080000:3.0:1767319130.334150:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319130.334153:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319130.334156:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985090432:0@lo:38:kworker.0 00000100:00080000:3.0:1767319130.334189:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319130.334192:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319130.334195:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985090688:0@lo:38:kworker.0 00000100:00100000:0.0:1767319130.334347:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985090304:0@lo:400:kworker.0 00000100:00100000:0.0:1767319130.334359:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6aded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985090560:0@lo:400:kworker.0 00000100:00100000:0.0:1767319130.843604:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319130.843628:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985091072:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319130.843701:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319130.843797:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091072 00000100:00100000:3.0:1767319130.843807:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091072 00000100:00100000:3.0:1767319130.843863:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2084 00000100:00100000:3.0:1767319130.843872:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985091072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319130.843902:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319130.843943:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6aded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985091072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (242us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319130.843950:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2084 00000100:00100000:0.0:1767319130.844036:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985091072:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319130.844451:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985091200:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319130.844481:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319130.844585:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091200 00000100:00100000:1.0:1767319130.844591:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091200 00000100:00100000:1.0:1767319130.844628:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2085 00000100:00100000:1.0:1767319130.844635:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985091200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319130.844653:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319130.844694:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985091200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 59us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319130.844701:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2085 00000100:00100000:0.0:1767319130.844762:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985091200:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319130.972975:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319131.995592:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319133.019417:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319134.043494:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319134.377857:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319134.377908:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319134.377999:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920543616 00000100:00100000:2.0:1767319134.378010:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920543616 00000100:00100000:2.0:1767319134.378079:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920543488 00000100:00100000:2.0:1767319134.378083:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920543488 00000100:00100000:2.0:1767319134.378103:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4436 00000100:00100000:2.0:1767319134.378113:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920543616:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319134.378216:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920543616:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 104us (368us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319134.378225:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4436 00000100:00100000:2.0:1767319134.378244:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4851 00000100:00100000:2.0:1767319134.378249:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6add880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920543488:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319134.378281:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319134.378333:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6add880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920543488:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 83us (424us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319134.378341:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4851 00000100:00080000:3.0:1767319134.378539:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319135.069140:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319135.452009:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319135.452019:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319135.452023:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319135.452028:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319135.452043:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319135.452053:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4378 00000100:00080000:0.0:1767319135.452058:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319135.452065:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319135.452106:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adc000 x1853163985091328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319135.452130:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319135.452134:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319135.452136:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319135.452139:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4378 00000100:00080000:0.0:1767319135.452141:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319135.452144:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319135.452161:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1f80 x1853163985091456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319135.452199:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985091328:0@lo:38:kworker.0 00000100:00080000:0.0:1767319135.452255:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319135.452259:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319135.452263:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319135.452268:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4378 00000100:00080000:0.0:1767319135.452271:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319135.452274:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319135.452298:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985091712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319135.452319:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767319135.452327:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319135.452329:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319135.452334:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767319135.452335:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985091584:0@lo:400:kworker.0 00000100:00080000:0.0:1767319135.452340:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4378 00000100:00080000:0.0:1767319135.452344:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319135.452349:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319135.452365:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74a80 x1853163985091968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319135.452410:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319135.452429:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985091456:0@lo:38:kworker.0 00000100:00100000:0.0:1767319135.452454:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985092224:0@lo:400:kworker.0 00000100:00100000:1.0:1767319135.452464:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319135.452474:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985091840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319135.452482:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319135.452517:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319135.452536:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985091712:0@lo:38:kworker.0 00000100:00100000:0.0:1767319135.452552:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319135.452564:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985092096:0@lo:400:kworker.0 00000100:00100000:2.0:1767319135.452570:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985092224 00000100:00100000:2.0:1767319135.452578:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985092224 00000100:00100000:2.0:1767319135.452636:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2923 00000100:00100000:2.0:1767319135.452646:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985092224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319135.456021:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319135.456437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319135.456968:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091328 00000100:00100000:0.0:1767319135.456982:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091328 00000100:00100000:0.0:1767319135.457025:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091584 00000100:00100000:0.0:1767319135.457028:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091584 00000100:00100000:1.0:1767319135.457388:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319135.457437:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091712 00000100:00100000:1.0:1767319135.457442:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091712 00000100:00100000:1.0:1767319135.457464:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091840 00000100:00100000:1.0:1767319135.457467:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091840 00000100:00100000:1.0:1767319135.457496:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4437 00000100:00100000:1.0:1767319135.457506:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985091840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319135.457568:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985091840:12345-0@lo:400:kworker.0 Request processed in 63us (181us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319135.457575:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4437 00000100:00100000:1.0:1767319135.457593:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4852 00000100:00100000:1.0:1767319135.457598:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985091328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319135.457619:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319135.457646:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985091328:12345-0@lo:38:kworker.0 Request processed in 47us (5328us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319135.457651:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4852 00000100:00100000:1.0:1767319135.457664:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4853 00000100:00100000:1.0:1767319135.457669:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985091712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319135.457677:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319135.457699:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985091712:12345-0@lo:38:kworker.0 Request processed in 29us (1270us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319135.457704:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4853 00000100:00100000:1.0:1767319135.457742:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985091840:0@lo:400:kworker.0 00000100:00100000:1.0:1767319135.457787:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985092096 00000100:00100000:1.0:1767319135.457791:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985092096 00000100:00100000:1.0:1767319135.457819:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2924 00000100:00100000:1.0:1767319135.457825:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985092096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319135.458727:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985091968:0@lo:38:kworker.0 00000100:00100000:3.0:1767319135.458808:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319135.458878:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319135.458886:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319135.458891:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985091328:0@lo:38:kworker.0 00000100:00080000:3.0:1767319135.458927:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319135.458931:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319135.458991:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985092096:12345-0@lo:400:kworker.0 Request processed in 1158us (2971us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319135.459004:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2924 00000100:00080000:1.0:1767319135.459041:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319135.459046:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319135.459049:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319135.459121:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985092096:0@lo:400:kworker.0 00000100:00100000:1.0:1767319135.459179:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091456 00000100:00100000:1.0:1767319135.459184:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091456 00000100:00100000:1.0:1767319135.459216:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1703 00000100:00100000:1.0:1767319135.459222:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985091456:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319135.459241:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319135.459309:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985091456:12345-0@lo:38:kworker.0 Request processed in 85us (6831us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319135.459315:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985091968 00000100:00100000:1.0:1767319135.459316:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1703 00000100:00100000:0.0:1767319135.459322:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985091968 00000100:00100000:0.0:1767319135.459340:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4438 00000100:00100000:1.0:1767319135.459347:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4854 00000100:00100000:0.0:1767319135.459350:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985091584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319135.459357:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985091968:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319135.459369:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319135.459412:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985091968:12345-0@lo:38:kworker.0 Request processed in 56us (603us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319135.459416:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985091584:12345-0@lo:400:kworker.0 Request processed in 67us (7005us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319135.459423:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4854 00000100:00080000:1.0:1767319135.459450:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319135.459478:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985091584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319135.459636:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985092224:12345-0@lo:400:kworker.0 Request processed in 6991us (7119us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319135.459645:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2923 00000100:00100000:3.0:1767319135.460066:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985091712:0@lo:38:kworker.0 00000100:00100000:1.0:1767319135.460070:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985092224:0@lo:400:kworker.0 00000100:00080000:3.0:1767319135.460108:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319135.460115:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319135.460119:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985091968:0@lo:38:kworker.0 00000100:00080000:3.0:1767319135.460150:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319135.460154:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319135.460157:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985091456:0@lo:38:kworker.0 00000100:00100000:0.0:1767319135.460967:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4438 00000100:00100000:1.0:1767319135.963437:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319135.963464:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985092352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319135.963547:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319135.963593:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985092352 00000100:00100000:1.0:1767319135.963597:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985092480:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319135.963608:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985092352 00000100:00100000:1.0:1767319135.963631:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319135.963689:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985092480 00000100:00100000:3.0:1767319135.963696:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985092480 00000100:00100000:1.0:1767319135.963700:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2086 00000100:00100000:1.0:1767319135.963720:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985092352:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319135.963739:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2087 00000100:00100000:3.0:1767319135.963754:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985092480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319135.963790:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:1.0:1767319135.963847:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319135.963880:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985092480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 129us (248us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319135.963969:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985092480:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319135.964031:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985092352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 320us (484us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319135.964197:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985092352:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319135.965281:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2087 00000100:00100000:1.0:1767319135.974265:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2086 00010000:00010000:1.0:1767319136.094917:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319137.115477:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319138.139656:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319138.483366:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319138.483403:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920543744 offset 0 length 520: 4 00000100:00100000:0.0:1767319138.483606:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544000 00000100:00100000:0.0:1767319138.483619:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544000 00000100:00100000:0.0:1767319138.483710:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4439 00000100:00100000:0.0:1767319138.483721:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920544000:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319138.483833:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920544000:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 113us (683us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319138.483843:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4439 00000100:00100000:2.0:1767319138.484020:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319138.484044:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920543872 00000100:00100000:2.0:1767319138.484048:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920543872 00000100:00100000:2.0:1767319138.484078:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4855 00000100:00100000:2.0:1767319138.484085:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920543872:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319138.484111:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319138.484144:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920543872:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 58us (124us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319138.484151:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4855 00000100:00080000:3.0:1767319138.484340:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319139.165965:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319139.548289:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319114/real 1767319114] req@ffff8ce1b524df80 x1853163985086080/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319139 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319139.548337:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b524df80 x1853163985086080/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319139 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319139.548363:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319139.548395:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319139.548399:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985086080:0@lo:250:kworker.0 00000100:00080000:2.0:1767319139.549102:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319139.549114:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319139.549118:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319139.549126:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4363 00000100:00080000:2.0:1767319139.549131:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319139.549139:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319139.549228:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633480 x1853163985092608/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319139.549259:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319139.549263:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319139.549264:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319139.549269:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4383 00000100:00080000:2.0:1767319139.549272:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319139.549275:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319139.549294:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985092736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319139.549303:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319139.549306:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319139.549308:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319139.549312:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4383 00000100:00080000:2.0:1767319139.549314:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319139.549318:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319139.549331:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc55c00 x1853163985092864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319139.549381:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319139.549386:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319139.549389:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319139.549392:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4383 00000100:00080000:2.0:1767319139.549395:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319139.549398:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319139.549765:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc56a00 x1853163985093120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319139.549787:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319139.549790:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319139.549792:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319139.549795:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4383 00000100:00080000:2.0:1767319139.549798:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319139.549802:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319139.549813:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc54000 x1853163985093376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319139.549881:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985093504:0@lo:400:kworker.0 00000100:00100000:1.0:1767319139.550136:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985092992:0@lo:400:kworker.0 00000100:00100000:1.0:1767319139.550255:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319139.550345:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319139.550351:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985093248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319139.550375:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319139.550623:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985092608:0@lo:250:kworker.0 00000400:00000100:0.0:1767319139.550652:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985092608 offset 0 length 520: 4 00000100:00100000:0.0:1767319139.550770:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985092736:0@lo:38:kworker.0 00000100:00100000:0.0:1767319139.550799:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319139.550822:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985092864:0@lo:38:kworker.0 00000100:00100000:0.0:1767319139.550847:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319139.550865:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985093120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319139.550886:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319139.551143:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985093632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319139.551214:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319139.551260:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985092992 00000100:00100000:3.0:1767319139.551270:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985092992 00000100:00100000:3.0:1767319139.551335:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985093248 00000100:00100000:3.0:1767319139.551338:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985093248 00000100:00100000:3.0:1767319139.551363:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985092736 00000100:00100000:3.0:1767319139.551365:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985092736 00000100:00100000:3.0:1767319139.551375:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985093120 00000100:00100000:3.0:1767319139.551377:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985093120 00000100:00100000:3.0:1767319139.551390:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4440 00000100:00100000:3.0:1767319139.551399:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985092992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319139.551815:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319139.551858:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985093632 00000100:00100000:2.0:1767319139.551862:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985093632 00000100:00100000:2.0:1767319139.551899:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985093504 00000100:00100000:2.0:1767319139.551902:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985093504 00000100:00100000:1.0:1767319139.552046:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4441 00000100:00100000:1.0:1767319139.552057:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985093248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319139.552127:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985093248:12345-0@lo:400:kworker.0 Request processed in 72us (1750us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319139.552134:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4441 00000100:00100000:1.0:1767319139.552158:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4856 00000100:00100000:1.0:1767319139.552174:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985092736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319139.552198:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319139.552241:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985092736:12345-0@lo:38:kworker.0 Request processed in 77us (1441us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319139.552247:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4856 00000100:00100000:1.0:1767319139.552260:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4857 00000100:00100000:1.0:1767319139.552264:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985093120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319139.552273:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319139.552296:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985093120:12345-0@lo:38:kworker.0 Request processed in 31us (1410us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319139.552302:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4857 00000100:00100000:1.0:1767319139.552324:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985092864 00000100:00100000:1.0:1767319139.552328:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985092864 00000100:00100000:1.0:1767319139.552352:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1704 00000100:00100000:1.0:1767319139.552356:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985092864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319139.552365:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319139.552391:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985092864:12345-0@lo:38:kworker.0 Request processed in 33us (1543us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319139.552397:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1704 00000100:00080000:0.0:1767319139.552556:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319139.552563:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319139.552567:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985093120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319139.552578:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985093376:0@lo:38:kworker.0 00000100:00100000:0.0:1767319139.552612:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319139.552653:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319139.552657:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319139.552661:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985092736:0@lo:38:kworker.0 00000100:00100000:3.0:1767319139.553110:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985092992:12345-0@lo:400:kworker.0 Request processed in 1711us (2855us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319139.553120:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4440 00000100:00100000:3.0:1767319139.553142:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985093376 00000100:00100000:3.0:1767319139.553146:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985093376 00000100:00100000:3.0:1767319139.553175:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4858 00000100:00100000:3.0:1767319139.553181:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985093376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319139.553193:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319139.553230:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985093376:12345-0@lo:38:kworker.0 Request processed in 47us (617us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319139.553235:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4858 00000100:00080000:3.0:1767319139.553263:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319139.553269:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319139.553272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319139.553528:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2925 00000100:00100000:2.0:1767319139.553543:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985093632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319139.553593:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985093632:12345-0@lo:400:kworker.0 Request processed in 56us (2379us total) trans 0 rc 0/0 00000100:00080000:0.0:1767319139.553657:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319139.553661:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319139.553813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985092864:0@lo:38:kworker.0 00000100:00080000:0.0:1767319139.553827:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319139.553830:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319139.553833:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985093376:0@lo:38:kworker.0 00000100:00100000:1.0:1767319139.555951:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985093248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319139.555976:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985092992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319139.556110:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2926 00000100:00100000:3.0:1767319139.556134:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985093504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319139.556273:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985093504:12345-0@lo:400:kworker.0 Request processed in 151us (4459us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319139.556282:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2926 00000100:00100000:3.0:1767319139.556346:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985093632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319139.556387:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985093504:0@lo:400:kworker.0 00000100:00100000:2.0:1767319139.559189:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2925 00010000:00010000:1.0:1767319140.189460:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319141.087840:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319141.088086:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985093888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319141.088379:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319141.088432:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985093760:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319141.088454:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319141.088551:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985093888 00000100:00100000:0.0:1767319141.088564:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985093888 00000100:00100000:0.0:1767319141.088642:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985093760 00000100:00100000:0.0:1767319141.088646:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985093760 00000100:00100000:0.0:1767319141.088672:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2088 00000100:00100000:0.0:1767319141.088681:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985093888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319141.088723:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319141.088777:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985093888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (397us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319141.088790:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2088 00000100:00100000:0.0:1767319141.088811:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2089 00000100:00100000:0.0:1767319141.088818:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985093760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319141.088834:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319141.088859:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985093760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 42us (404us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319141.088865:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2089 00000100:00080000:2.0:1767319141.089373:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319141.089379:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319141.089432:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985093888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319141.089451:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985093760:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319141.211743:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319142.235673:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319143.259683:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319143.597613:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319143.597716:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319143.597907:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544256 00000100:00100000:3.0:1767319143.597917:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544256 00000100:00100000:3.0:1767319143.598065:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544128 00000100:00100000:3.0:1767319143.598069:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544128 00000100:00100000:3.0:1767319143.598088:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4442 00000100:00100000:3.0:1767319143.598098:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920544256:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319143.598209:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920544256:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 113us (605us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319143.598217:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4442 00000100:00100000:3.0:1767319143.598236:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4859 00000100:00100000:3.0:1767319143.598240:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920544128:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319143.598260:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319143.598284:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920544128:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (569us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319143.598290:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4859 00000100:00080000:2.0:1767319143.599213:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319144.283587:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319144.667536:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319144.667547:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319144.667550:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319144.667554:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319144.667558:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319144.667567:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4387 00000100:00080000:3.0:1767319144.667572:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319144.667579:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319144.667620:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163985094016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319144.667648:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319144.667651:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319144.667653:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319144.667657:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4387 00000100:00080000:3.0:1767319144.667660:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319144.667663:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319144.667677:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a300 x1853163985094144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319144.667715:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319144.667719:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319144.667720:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319144.667723:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4387 00000100:00080000:3.0:1767319144.667726:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319144.667729:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319144.667742:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3100 x1853163985094400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319144.667759:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319144.667762:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319144.667764:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319144.667767:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4387 00000100:00080000:3.0:1767319144.667769:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319144.667773:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319144.667784:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985094656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319144.667831:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985094784:0@lo:400:kworker.0 00000100:00100000:3.0:1767319144.667898:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319144.667957:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319144.667963:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985094912:0@lo:400:kworker.0 00000100:00100000:3.0:1767319144.667987:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319144.668070:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094784 00000100:00100000:1.0:1767319144.668078:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094784 00000100:00100000:1.0:1767319144.668145:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094912 00000100:00100000:1.0:1767319144.668149:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094912 00000100:00100000:1.0:1767319144.668197:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2927 00000100:00100000:1.0:1767319144.668207:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985094784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668261:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985094784:12345-0@lo:400:kworker.0 Request processed in 55us (362us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319144.668267:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2927 00000100:00100000:1.0:1767319144.668284:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2928 00000100:00100000:1.0:1767319144.668291:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985094912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668317:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985094912:12345-0@lo:400:kworker.0 Request processed in 26us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319144.668322:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2928 00000100:00100000:1.0:1767319144.668340:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985094528:0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668369:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319144.668394:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319144.668401:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985094272:0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668422:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319144.668448:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094528 00000100:00100000:1.0:1767319144.668452:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094528 00000100:00100000:1.0:1767319144.668479:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094272 00000100:00100000:1.0:1767319144.668483:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094272 00000100:00100000:1.0:1767319144.668503:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4443 00000100:00100000:1.0:1767319144.668508:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985094528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668543:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985094528:12345-0@lo:400:kworker.0 Request processed in 34us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319144.668549:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4443 00000100:00100000:1.0:1767319144.668561:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4444 00000100:00100000:1.0:1767319144.668566:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985094272:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668592:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985094272:12345-0@lo:400:kworker.0 Request processed in 26us (170us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319144.668598:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4444 00000100:00100000:1.0:1767319144.668633:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985094912:0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668642:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985094784:0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668669:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985094272:0@lo:400:kworker.0 00000100:00100000:1.0:1767319144.668677:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985094528:0@lo:400:kworker.0 00000100:00080000:3.0:1767319144.668757:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319144.668762:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319144.668764:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319144.668910:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985094016:0@lo:38:kworker.0 00000100:00100000:0.0:1767319144.668944:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319144.668977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985094144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319144.668998:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319144.669027:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985094400:0@lo:38:kworker.0 00000100:00100000:0.0:1767319144.669045:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319144.669068:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985094656:0@lo:38:kworker.0 00000100:00100000:0.0:1767319144.669088:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319144.669127:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094016 00000100:00100000:0.0:1767319144.669132:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094016 00000100:00100000:0.0:1767319144.669151:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094400 00000100:00100000:0.0:1767319144.669153:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094400 00000100:00100000:0.0:1767319144.669184:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094656 00000100:00100000:0.0:1767319144.669187:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094656 00000100:00100000:0.0:1767319144.669200:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4860 00000100:00100000:0.0:1767319144.669204:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985094016:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319144.669221:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319144.669255:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985094016:12345-0@lo:38:kworker.0 Request processed in 48us (309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319144.669261:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4860 00000100:00100000:0.0:1767319144.669274:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4861 00000100:00100000:0.0:1767319144.669278:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985094400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319144.669286:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319144.669309:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985094400:12345-0@lo:38:kworker.0 Request processed in 29us (262us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319144.669315:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4861 00000100:00100000:0.0:1767319144.669325:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4862 00000100:00100000:0.0:1767319144.669328:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985094656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319144.669336:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319144.669357:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985094656:12345-0@lo:38:kworker.0 Request processed in 26us (268us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319144.669362:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4862 00000100:00080000:0.0:1767319144.669385:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319144.669390:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319144.669393:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985094016:0@lo:38:kworker.0 00000100:00080000:0.0:1767319144.669400:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319144.669403:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319144.669406:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985094400:0@lo:38:kworker.0 00000100:00080000:0.0:1767319144.669411:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319144.669414:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319144.669417:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985094656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319144.669753:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985094144 00000100:00100000:1.0:1767319144.669758:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985094144 00000100:00100000:1.0:1767319144.669790:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1705 00000100:00100000:1.0:1767319144.669795:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985094144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319144.669806:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319144.669847:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985094144:12345-0@lo:38:kworker.0 Request processed in 49us (847us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319144.669851:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1705 00000100:00080000:0.0:1767319144.670395:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319144.670401:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319144.670404:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985094144:0@lo:38:kworker.0 00010000:00010000:0.0:1767319145.307476:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319146.204284:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985095040:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319146.204369:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319146.204481:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095040 00000100:00100000:3.0:1767319146.204489:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095040 00000100:00100000:3.0:1767319146.204555:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2090 00000100:00100000:3.0:1767319146.204565:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985095040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319146.204599:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319146.204655:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985095040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (286us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319146.204663:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2090 00000100:00080000:2.0:1767319146.204732:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319146.204769:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985095040:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319146.205812:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319146.205829:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985095168:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319146.205892:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319146.206085:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095168 00000100:00100000:3.0:1767319146.206093:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095168 00000100:00100000:3.0:1767319146.206156:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2091 00000100:00100000:3.0:1767319146.206206:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985095168:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319146.206231:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319146.206273:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985095168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (379us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319146.206280:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2091 00000100:00080000:2.0:1767319146.206447:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319146.206493:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985095168:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319146.332478:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319147.357218:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319148.383119:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319148.713855:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319148.713961:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544512 00000100:00100000:2.0:1767319148.713972:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544512 00000100:00100000:2.0:1767319148.714077:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4445 00000100:00100000:2.0:1767319148.714089:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920544512:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319148.714223:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920544512:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 136us (377us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319148.714232:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4445 00000100:00080000:3.0:1767319148.714449:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319148.719332:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319148.729272:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544384 00000100:00100000:1.0:1767319148.729290:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544384 00000100:00100000:1.0:1767319148.729368:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4863 00000100:00100000:1.0:1767319148.729377:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920544384:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319148.729403:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319148.729641:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920544384:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 262us (10319us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319148.729650:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4863 00010000:00010000:2.0:1767319149.406997:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319149.787385:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319149.787395:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319149.787397:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319149.787402:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319149.787406:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319149.787415:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4392 00000100:00080000:0.0:1767319149.787420:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319149.787427:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319149.787471:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a300 x1853163985095296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319149.787513:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319149.787517:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319149.787519:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319149.787522:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4392 00000100:00080000:0.0:1767319149.787525:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319149.787529:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319149.787556:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163985095424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319149.787602:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319149.787606:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319149.787608:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319149.787611:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4392 00000100:00080000:0.0:1767319149.787613:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319149.787617:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319149.787628:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6d80 x1853163985095680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319149.787645:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319149.787647:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319149.787649:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319149.787652:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4392 00000100:00080000:0.0:1767319149.787654:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319149.787657:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319149.787668:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6680 x1853163985095936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319149.787717:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985096192:0@lo:400:kworker.0 00000100:00100000:0.0:1767319149.787777:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319149.787836:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319149.787843:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985096064:0@lo:400:kworker.0 00000100:00100000:0.0:1767319149.787866:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319149.787892:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096192 00000100:00100000:0.0:1767319149.787899:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096192 00000100:00100000:0.0:1767319149.787956:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096064 00000100:00100000:0.0:1767319149.787960:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096064 00000100:00100000:0.0:1767319149.787981:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2929 00000100:00100000:0.0:1767319149.787989:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985096192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319149.788047:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985096192:12345-0@lo:400:kworker.0 Request processed in 59us (270us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319149.788055:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2929 00000100:00100000:0.0:1767319149.788072:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2930 00000100:00100000:0.0:1767319149.788079:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985096064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319149.788359:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985095808:0@lo:400:kworker.0 00000100:00100000:3.0:1767319149.788390:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319149.788415:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319149.788421:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985095552:0@lo:400:kworker.0 00000100:00100000:3.0:1767319149.788461:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319149.788488:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319149.788493:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319149.788788:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985096064:12345-0@lo:400:kworker.0 Request processed in 708us (919us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319149.788796:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2930 00000100:00100000:2.0:1767319149.788898:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985095296:0@lo:38:kworker.0 00000100:00100000:2.0:1767319149.788941:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319149.788969:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985095424:0@lo:38:kworker.0 00000100:00100000:2.0:1767319149.788991:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319149.789026:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985095680:0@lo:38:kworker.0 00000100:00100000:2.0:1767319149.789047:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319149.789064:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985095936:0@lo:38:kworker.0 00000100:00100000:2.0:1767319149.789084:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319149.789127:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095808 00000100:00100000:2.0:1767319149.789132:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095808 00000100:00100000:2.0:1767319149.789195:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095552 00000100:00100000:2.0:1767319149.789198:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095552 00000100:00100000:2.0:1767319149.789214:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095296 00000100:00100000:2.0:1767319149.789217:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095296 00000100:00100000:2.0:1767319149.789225:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095680 00000100:00100000:2.0:1767319149.789227:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095680 00000100:00100000:2.0:1767319149.789234:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095936 00000100:00100000:2.0:1767319149.789236:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095936 00000100:00100000:2.0:1767319149.789245:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4446 00000100:00100000:2.0:1767319149.789252:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985095808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319149.789290:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985095808:12345-0@lo:400:kworker.0 Request processed in 38us (899us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319149.789296:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4446 00000100:00100000:2.0:1767319149.789309:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4447 00000100:00100000:2.0:1767319149.789315:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985095552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319149.789340:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985095552:12345-0@lo:400:kworker.0 Request processed in 26us (879us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319149.789346:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4447 00000100:00100000:2.0:1767319149.789357:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4864 00000100:00100000:2.0:1767319149.789361:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985095296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319149.789377:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319149.789407:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985095296:12345-0@lo:38:kworker.0 Request processed in 44us (465us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319149.789413:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4864 00000100:00100000:2.0:1767319149.789424:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4865 00000100:00100000:2.0:1767319149.789427:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985095680:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319149.789436:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319149.789463:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985095680:12345-0@lo:38:kworker.0 Request processed in 33us (414us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319149.789468:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4865 00000100:00100000:2.0:1767319149.789478:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4866 00000100:00100000:2.0:1767319149.789482:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985095936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319149.789490:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319149.789508:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985095936:12345-0@lo:38:kworker.0 Request processed in 25us (424us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319149.789513:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4866 00000100:00100000:2.0:1767319149.789534:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985095424 00000100:00100000:2.0:1767319149.789543:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985095424 00000100:00100000:2.0:1767319149.789561:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1706 00000100:00100000:2.0:1767319149.789565:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985095424:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319149.789574:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319149.789598:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985095424:12345-0@lo:38:kworker.0 Request processed in 32us (606us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319149.789604:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1706 00000100:00080000:2.0:1767319149.789618:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319149.789647:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985095552:0@lo:400:kworker.0 00000100:00100000:2.0:1767319149.789658:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985095808:0@lo:400:kworker.0 00000100:00080000:2.0:1767319149.789689:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319149.789693:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319149.789696:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985095296:0@lo:38:kworker.0 00000100:00080000:2.0:1767319149.789704:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319149.789707:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319149.789709:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985095424:0@lo:38:kworker.0 00000100:00080000:2.0:1767319149.789715:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319149.789718:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319149.789721:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985095680:0@lo:38:kworker.0 00000100:00080000:2.0:1767319149.789727:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319149.789730:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319149.789733:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985095936:0@lo:38:kworker.0 00000100:00100000:3.0:1767319149.791394:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985096064:0@lo:400:kworker.0 00000100:00100000:3.0:1767319149.791411:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985096192:0@lo:400:kworker.0 00010000:00010000:2.0:1767319150.428132:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319151.323533:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319151.323559:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985096448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319151.323627:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319151.323747:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985096320:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319151.323765:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319151.323908:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096448 00000100:00100000:1.0:1767319151.323917:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096448 00000100:00100000:3.0:1767319151.323943:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096320 00000100:00100000:3.0:1767319151.323949:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096320 00000100:00080000:2.0:1767319151.324118:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319151.324123:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319151.324376:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2092 00000100:00100000:1.0:1767319151.324385:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985096448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319151.324410:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319151.324473:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985096448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (844us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319151.324480:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2092 00000100:00100000:1.0:1767319151.324496:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2093 00000100:00100000:1.0:1767319151.324502:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985096320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319151.324518:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319151.324544:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985096320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (777us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319151.324550:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2093 00000100:00100000:2.0:1767319151.324598:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985096448:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319151.324610:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985096320:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319151.451682:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319152.475411:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319153.500617:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319153.835984:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319153.836059:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319153.836189:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544640 00000100:00100000:2.0:1767319153.836199:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544640 00000100:00100000:2.0:1767319153.836242:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544768 00000100:00100000:2.0:1767319153.836245:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544768 00000100:00100000:2.0:1767319153.836289:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4448 00000100:00100000:2.0:1767319153.836299:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920544768:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319153.836370:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920544768:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 72us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319153.836378:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4448 00000100:00100000:2.0:1767319153.836397:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4867 00000100:00100000:2.0:1767319153.836401:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920544640:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319153.836420:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319153.836445:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920544640:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (472us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319153.836451:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4867 00000100:00080000:2.0:1767319153.836469:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319154.525462:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319154.907342:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319154.907352:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319154.907355:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319154.907360:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319154.907364:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319154.907372:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4398 00000100:00080000:2.0:1767319154.907377:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319154.907384:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319154.907422:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c000 x1853163985096576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319154.907438:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319154.907441:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319154.907443:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319154.907446:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4398 00000100:00080000:2.0:1767319154.907449:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319154.907452:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319154.907471:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163985096704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319154.907512:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319154.907515:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319154.907517:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319154.907521:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4398 00000100:00080000:2.0:1767319154.907523:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319154.907527:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319154.907539:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5c00 x1853163985096960/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319154.907542:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767319154.907559:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767319154.907560:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985096832:0@lo:400:kworker.0 00000100:00080000:2.0:1767319154.907565:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319154.907578:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319154.907582:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4398 00000100:00080000:2.0:1767319154.907585:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319154.907590:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319154.907602:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319154.907603:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6680 x1853163985097216/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319154.907617:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985097088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319154.907632:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319154.907651:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319154.907663:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985097344:0@lo:400:kworker.0 00000100:00100000:1.0:1767319154.907724:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096832 00000100:00100000:1.0:1767319154.907732:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096832 00000100:00100000:1.0:1767319154.907777:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097088 00000100:00100000:1.0:1767319154.907780:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097088 00000100:00100000:1.0:1767319154.907804:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4449 00000100:00100000:1.0:1767319154.907813:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985096832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319154.907883:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319154.907908:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319154.907943:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985097472:0@lo:400:kworker.0 00000100:00100000:1.0:1767319154.907956:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985096832:12345-0@lo:400:kworker.0 Request processed in 142us (322us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319154.907965:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4449 00000100:00100000:1.0:1767319154.907984:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4450 00000100:00100000:1.0:1767319154.907990:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985097088:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319154.907996:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319154.908029:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985097088:12345-0@lo:400:kworker.0 Request processed in 39us (379us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319154.908036:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4450 00000100:00100000:2.0:1767319154.908036:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097344 00000100:00100000:2.0:1767319154.908041:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097344 00000100:00100000:1.0:1767319154.908079:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985096832:0@lo:400:kworker.0 00000100:00100000:0.0:1767319154.908269:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985096576:0@lo:38:kworker.0 00000100:00100000:0.0:1767319154.908300:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319154.908325:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985096704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319154.908346:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319154.908368:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985096960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319154.908388:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319154.908407:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985097216:0@lo:38:kworker.0 00000100:00100000:0.0:1767319154.908428:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319154.909580:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985097088:0@lo:400:kworker.0 00000100:00100000:0.0:1767319154.910950:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096704 00000100:00100000:0.0:1767319154.910963:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096704 00000100:00100000:0.0:1767319154.911006:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1707 00000100:00100000:0.0:1767319154.911013:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985096704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319154.911032:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319154.911093:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985096704:12345-0@lo:38:kworker.0 Request processed in 80us (2746us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319154.911098:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1707 00000100:00100000:0.0:1767319154.911119:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096576 00000100:00100000:0.0:1767319154.911122:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096576 00000100:00100000:0.0:1767319154.911135:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985096960 00000100:00100000:0.0:1767319154.911137:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985096960 00000100:00100000:0.0:1767319154.911141:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097216 00000100:00100000:0.0:1767319154.911143:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097216 00000100:00100000:0.0:1767319154.911148:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4868 00000100:00100000:0.0:1767319154.911151:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985096576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319154.911159:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319154.911204:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985096576:12345-0@lo:38:kworker.0 Request processed in 51us (2904us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319154.911211:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4868 00000100:00100000:0.0:1767319154.911222:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4869 00000100:00100000:0.0:1767319154.911225:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985096960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319154.911233:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319154.911253:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985096960:12345-0@lo:38:kworker.0 Request processed in 27us (2865us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319154.911257:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4869 00000100:00100000:0.0:1767319154.911264:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4870 00000100:00100000:0.0:1767319154.911266:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985097216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319154.911272:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319154.911285:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985097216:12345-0@lo:38:kworker.0 Request processed in 18us (2858us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319154.911288:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4870 00000100:00080000:0.0:1767319154.911313:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319154.911317:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319154.911321:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985096576:0@lo:38:kworker.0 00000100:00080000:0.0:1767319154.911331:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319154.911335:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319154.911338:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985096704:0@lo:38:kworker.0 00000100:00080000:0.0:1767319154.911346:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319154.911349:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319154.911352:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985096960:0@lo:38:kworker.0 00000100:00080000:0.0:1767319154.911359:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319154.911362:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319154.911364:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985097216:0@lo:38:kworker.0 00000100:00080000:1.0:1767319154.911394:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319154.911404:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319154.911429:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097472 00000100:00100000:0.0:1767319154.911432:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097472 00000100:00100000:0.0:1767319154.911486:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2931 00000100:00100000:0.0:1767319154.911493:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985097344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319154.911537:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985097344:12345-0@lo:400:kworker.0 Request processed in 45us (3654us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319154.911543:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2931 00000100:00100000:0.0:1767319154.911557:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2932 00000100:00100000:0.0:1767319154.911563:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985097472:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319154.911589:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319154.911925:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985097472:12345-0@lo:400:kworker.0 Request processed in 351us (3920us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319154.911931:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2932 00000100:00100000:0.0:1767319154.911992:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985097472:0@lo:400:kworker.0 00000100:00100000:0.0:1767319154.912004:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985097344:0@lo:400:kworker.0 00010000:00010000:2.0:1767319155.551400:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319156.444141:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985097600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319156.444227:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319156.444325:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097600 00000100:00100000:1.0:1767319156.444344:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097600 00000100:00100000:1.0:1767319156.444409:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2094 00000100:00100000:1.0:1767319156.444418:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985097600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319156.444447:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319156.444493:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985097600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319156.444500:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2094 00000100:00080000:2.0:1767319156.444566:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319156.444602:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985097600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319156.446029:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319156.446045:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985097728:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319156.446104:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319156.446269:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097728 00000100:00100000:1.0:1767319156.446275:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097728 00000100:00100000:1.0:1767319156.446340:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2095 00000100:00100000:1.0:1767319156.446348:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985097728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319156.446370:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319156.446409:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985097728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319156.446416:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2095 00000100:00080000:2.0:1767319156.446511:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319156.446539:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985097728:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319156.571642:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319157.595661:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319158.619704:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319158.956724:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319158.956776:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319158.956897:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920545024 00000100:00100000:1.0:1767319158.956908:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920545024 00000100:00100000:1.0:1767319158.956988:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920544896 00000100:00100000:1.0:1767319158.956991:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920544896 00000100:00100000:1.0:1767319158.957011:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4451 00000100:00100000:1.0:1767319158.957021:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920545024:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319158.957100:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920545024:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (386us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319158.957108:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4451 00000100:00100000:1.0:1767319158.957127:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4871 00000100:00100000:1.0:1767319158.957131:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920544896:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319158.957158:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319158.957219:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920544896:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 86us (443us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319158.957226:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4871 00000100:00080000:3.0:1767319158.961033:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319159.643893:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319160.030679:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319160.030691:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319160.030694:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319160.030698:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319160.030703:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319160.030710:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4403 00000100:00080000:3.0:1767319160.030715:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319160.030722:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319160.030762:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2d80 x1853163985097856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319160.030787:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319160.030798:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319160.030799:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319160.030803:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4403 00000100:00080000:3.0:1767319160.030806:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319160.030809:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319160.030833:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985097984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319160.030870:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319160.030874:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319160.030876:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319160.030880:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4403 00000100:00080000:3.0:1767319160.030882:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319160.030885:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319160.030899:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0e00 x1853163985098240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319160.030914:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319160.030916:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319160.030918:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319160.030921:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4403 00000100:00080000:3.0:1767319160.030923:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319160.030926:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319160.030934:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1500 x1853163985098496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319160.030979:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985098752:0@lo:400:kworker.0 00000100:00100000:3.0:1767319160.031044:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319160.031109:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319160.031115:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985098624:0@lo:400:kworker.0 00000100:00100000:3.0:1767319160.031141:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319160.031196:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985098752 00000100:00100000:3.0:1767319160.031204:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985098752 00000100:00100000:3.0:1767319160.031257:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985098624 00000100:00100000:3.0:1767319160.031261:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985098624 00000100:00100000:3.0:1767319160.031282:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2933 00000100:00100000:3.0:1767319160.031290:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985098752:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319160.031337:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985098752:12345-0@lo:400:kworker.0 Request processed in 47us (291us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319160.031344:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2933 00000100:00100000:3.0:1767319160.031360:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2934 00000100:00100000:3.0:1767319160.031366:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985098624:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319160.031391:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985098624:12345-0@lo:400:kworker.0 Request processed in 26us (250us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319160.031397:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2934 00000100:00080000:3.0:1767319160.031427:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319160.031431:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319160.031460:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985098624:0@lo:400:kworker.0 00000100:00100000:3.0:1767319160.031471:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985098752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319160.031558:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985098368:0@lo:400:kworker.0 00000100:00100000:2.0:1767319160.031590:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319160.031625:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319160.031632:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985098112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319160.031654:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319160.031687:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985098368 00000100:00100000:2.0:1767319160.031692:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985098368 00000100:00100000:2.0:1767319160.031722:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985098112 00000100:00100000:2.0:1767319160.031726:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985098112 00000100:00100000:2.0:1767319160.031747:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4452 00000100:00100000:2.0:1767319160.031754:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985098368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319160.031788:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985098368:12345-0@lo:400:kworker.0 Request processed in 34us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319160.031794:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4452 00000100:00100000:2.0:1767319160.031807:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4453 00000100:00100000:2.0:1767319160.031813:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985098112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319160.031851:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985098112:12345-0@lo:400:kworker.0 Request processed in 39us (196us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319160.031857:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4453 00000100:00080000:2.0:1767319160.031872:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319160.031888:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985098112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319160.031899:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985098368:0@lo:400:kworker.0 00000100:00100000:1.0:1767319160.032670:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985097856:0@lo:38:kworker.0 00000100:00100000:1.0:1767319160.032717:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319160.032756:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985097984:0@lo:38:kworker.0 00000100:00100000:1.0:1767319160.032780:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319160.032800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985098240:0@lo:38:kworker.0 00000100:00100000:1.0:1767319160.032831:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319160.032854:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985098496:0@lo:38:kworker.0 00000100:00100000:1.0:1767319160.032873:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319160.032905:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097856 00000100:00100000:1.0:1767319160.032911:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097856 00000100:00100000:1.0:1767319160.032932:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985098240 00000100:00100000:1.0:1767319160.032935:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985098240 00000100:00100000:1.0:1767319160.032943:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985098496 00000100:00100000:1.0:1767319160.032945:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985098496 00000100:00100000:1.0:1767319160.032955:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4872 00000100:00100000:1.0:1767319160.032961:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985097856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319160.032980:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319160.033016:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985097856:12345-0@lo:38:kworker.0 Request processed in 54us (298us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319160.033024:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4872 00000100:00100000:1.0:1767319160.033038:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4873 00000100:00100000:1.0:1767319160.033041:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985098240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319160.033050:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319160.033072:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985098240:12345-0@lo:38:kworker.0 Request processed in 29us (240us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319160.033078:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4873 00000100:00100000:1.0:1767319160.033101:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4874 00000100:00100000:1.0:1767319160.033105:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985098496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319160.033114:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319160.033133:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985098496:12345-0@lo:38:kworker.0 Request processed in 27us (260us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319160.033138:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4874 00000100:00080000:1.0:1767319160.033190:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319160.033195:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319160.033199:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985097856:0@lo:38:kworker.0 00000100:00080000:1.0:1767319160.033208:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319160.033211:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319160.033214:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985098240:0@lo:38:kworker.0 00000100:00080000:1.0:1767319160.033222:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319160.033225:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319160.033228:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985098496:0@lo:38:kworker.0 00000100:00100000:0.0:1767319160.033567:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985097984 00000100:00100000:0.0:1767319160.033573:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985097984 00000100:00100000:0.0:1767319160.033606:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1708 00000100:00100000:0.0:1767319160.033611:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985097984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319160.033622:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319160.033657:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985097984:12345-0@lo:38:kworker.0 Request processed in 44us (876us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319160.033662:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1708 00000100:00080000:1.0:1767319160.034137:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319160.034144:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319160.034147:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985097984:0@lo:38:kworker.0 00010000:00010000:1.0:1767319160.667466:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319161.563664:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319161.563691:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985098880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319161.563757:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319161.563879:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985098880 00000100:00100000:3.0:1767319161.563888:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985098880 00000100:00100000:3.0:1767319161.563959:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2096 00000100:00100000:3.0:1767319161.563969:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985098880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319161.563996:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319161.564041:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985098880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (282us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319161.564052:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2096 00000100:00100000:2.0:1767319161.564488:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985098880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319161.564631:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985099008:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319161.564659:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319161.564727:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985099008 00000100:00100000:1.0:1767319161.564732:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985099008 00000100:00100000:1.0:1767319161.564766:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2097 00000100:00100000:1.0:1767319161.564773:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985099008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319161.564801:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319161.564846:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985099008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (186us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319161.564853:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2097 00000100:00100000:2.0:1767319161.564920:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985099008:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319161.691624:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319162.716313:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319163.739452:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319164.074669:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319164.074719:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319164.074775:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920545408 offset 0 length 520: 4 00000100:00100000:2.0:1767319164.075486:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920545280 00000100:00100000:2.0:1767319164.075497:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920545280 00000100:00100000:2.0:1767319164.075582:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920545152 00000100:00100000:2.0:1767319164.075586:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920545152 00000100:00100000:2.0:1767319164.075605:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4454 00000100:00100000:2.0:1767319164.075616:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920545280:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319164.075695:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920545280:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (1037us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319164.075703:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4454 00000100:00100000:2.0:1767319164.075721:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4875 00000100:00100000:2.0:1767319164.075725:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920545152:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319164.075758:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319164.075785:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920545152:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 57us (1066us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319164.075791:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4875 00000100:00080000:3.0:1767319164.076909:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319164.123829:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319139/real 1767319139] req@ffff8ce188633480 x1853163985092608/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319164 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319164.123874:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188633480 x1853163985092608/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319164 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319164.123898:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319164.123930:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319164.123935:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985092608:0@lo:250:kworker.0 00000100:00080000:3.0:1767319164.124146:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319164.124152:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319164.124155:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319164.124160:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4387 00000100:00080000:3.0:1767319164.124183:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319164.124191:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319164.124223:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0000 x1853163985099136/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319164.124249:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319164.124253:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319164.124255:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319164.124259:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4408 00000100:00080000:3.0:1767319164.124262:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319164.124266:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319164.124281:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1880 x1853163985099264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319164.124289:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319164.124292:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319164.124294:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319164.124297:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4408 00000100:00080000:3.0:1767319164.124301:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319164.124304:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319164.124315:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3100 x1853163985099392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319164.124392:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985099520:0@lo:400:kworker.0 00000100:00100000:2.0:1767319164.124455:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319164.124565:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985099136:0@lo:250:kworker.0 00000400:00000100:1.0:1767319164.124591:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985099136 offset 0 length 520: 4 00000100:00100000:1.0:1767319164.124611:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985099264:0@lo:38:kworker.0 00000100:00100000:1.0:1767319164.124643:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319164.124661:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985099392:0@lo:38:kworker.0 00000100:00100000:1.0:1767319164.124682:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319164.124715:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985099392 00000100:00100000:1.0:1767319164.124722:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985099392 00000100:00100000:1.0:1767319164.124768:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1709 00000100:00100000:1.0:1767319164.124775:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985099392:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319164.124795:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319164.124834:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985099392:12345-0@lo:38:kworker.0 Request processed in 59us (150us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319164.124841:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1709 00000100:00100000:1.0:1767319164.124876:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985099520 00000100:00100000:1.0:1767319164.124880:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985099520 00000100:00100000:1.0:1767319164.124918:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985099264 00000100:00100000:1.0:1767319164.124921:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985099264 00000100:00100000:1.0:1767319164.124939:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4455 00000100:00100000:1.0:1767319164.124945:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985099520:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319164.125000:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319164.125004:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319164.125007:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319164.125011:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4408 00000100:00080000:3.0:1767319164.125015:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319164.125019:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319164.125033:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985099648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319164.125059:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4876 00000100:00100000:0.0:1767319164.125066:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985099264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319164.125078:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319164.125109:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985099264:12345-0@lo:38:kworker.0 Request processed in 41us (464us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319164.125115:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4876 00000100:00100000:0.0:1767319164.125148:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985099648:0@lo:38:kworker.0 00000100:00100000:0.0:1767319164.125186:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319164.125211:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319164.125215:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319164.125219:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985099264:0@lo:38:kworker.0 00000100:00080000:0.0:1767319164.125226:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319164.125229:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319164.125232:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985099392:0@lo:38:kworker.0 00000100:00100000:0.0:1767319164.125253:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985099648 00000100:00100000:0.0:1767319164.125256:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985099648 00000100:00100000:0.0:1767319164.125273:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4877 00000100:00100000:0.0:1767319164.125277:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985099648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319164.125286:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319164.125311:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985099648:12345-0@lo:38:kworker.0 Request processed in 33us (125us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319164.125317:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4877 00000100:00080000:0.0:1767319164.125336:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319164.125340:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319164.125342:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985099648:0@lo:38:kworker.0 00000100:00080000:0.0:1767319164.125362:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319164.125386:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985099776:0@lo:400:kworker.0 00000100:00100000:0.0:1767319164.125408:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319164.125432:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985099776 00000100:00100000:0.0:1767319164.125436:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985099776 00000100:00100000:0.0:1767319164.125462:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4456 00000100:00100000:0.0:1767319164.125468:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985099776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319164.125513:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985099776:12345-0@lo:400:kworker.0 Request processed in 45us (104us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319164.125519:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4456 00000100:00080000:0.0:1767319164.125532:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319164.125548:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985099776:0@lo:400:kworker.0 00000100:00100000:1.0:1767319164.127050:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985099520:12345-0@lo:400:kworker.0 Request processed in 2100us (2590us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319164.127062:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4455 00000100:00080000:3.0:1767319164.127137:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319164.127142:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319164.127145:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319164.127149:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4408 00000100:00080000:3.0:1767319164.127153:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319164.127157:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319164.127204:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0e00 x1853163985099904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319164.127271:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985100160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319164.127314:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319164.127338:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319164.127343:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985100032:0@lo:400:kworker.0 00000100:00100000:3.0:1767319164.127364:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319164.127387:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100160 00000100:00100000:3.0:1767319164.127393:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100160 00000100:00100000:3.0:1767319164.127432:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100032 00000100:00100000:3.0:1767319164.127435:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100032 00000100:00100000:3.0:1767319164.127455:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2935 00000100:00100000:3.0:1767319164.127461:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985100160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319164.127511:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985100160:12345-0@lo:400:kworker.0 Request processed in 50us (196us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319164.127517:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2935 00000100:00100000:3.0:1767319164.127529:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2936 00000100:00100000:3.0:1767319164.127547:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985100032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319164.127573:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985100032:12345-0@lo:400:kworker.0 Request processed in 38us (208us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319164.127579:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2936 00000100:00100000:0.0:1767319164.127594:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985099904:0@lo:38:kworker.0 00000100:00080000:3.0:1767319164.127606:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319164.127611:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319164.127632:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319164.127789:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985100032:0@lo:400:kworker.0 00000100:00100000:3.0:1767319164.127803:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985100160:0@lo:400:kworker.0 00000100:00100000:1.0:1767319164.127874:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985099904 00000100:00100000:1.0:1767319164.127879:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985099904 00000100:00100000:1.0:1767319164.127908:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4878 00000100:00100000:1.0:1767319164.127913:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985099904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319164.127929:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319164.127974:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985099904:12345-0@lo:38:kworker.0 Request processed in 60us (342us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319164.127981:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4878 00000100:00080000:3.0:1767319164.128051:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319164.128057:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319164.128060:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985099904:0@lo:38:kworker.0 00000100:00100000:2.0:1767319164.128190:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985099520:0@lo:400:kworker.0 00010000:00010000:1.0:1767319164.763681:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319165.787577:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319166.683447:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985100416:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319166.683465:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985100288:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319166.683526:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319166.683562:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100288 00000100:00100000:1.0:1767319166.683569:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100288 00000100:00100000:2.0:1767319166.683585:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319166.683629:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100416 00000100:00100000:2.0:1767319166.683633:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100416 00000100:00100000:1.0:1767319166.683637:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2098 00000100:00100000:1.0:1767319166.683649:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985100288:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319166.683688:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319166.683746:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985100288:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (220us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319166.683753:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2098 00000100:00100000:1.0:1767319166.683770:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2099 00000100:00100000:1.0:1767319166.683776:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985100416:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319166.683792:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319166.683821:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985100416:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (236us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319166.683827:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2099 00000100:00100000:3.0:1767319166.684561:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985100288:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319166.684600:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985100416:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319166.684623:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319166.684629:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319166.811506:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319167.835588:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319168.859524:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319169.199956:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319169.200006:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319169.200026:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920545664 00000100:00100000:1.0:1767319169.200036:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920545664 00000100:00100000:0.0:1767319169.200198:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920545536 00000100:00100000:0.0:1767319169.200204:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920545536 00000100:00100000:0.0:1767319169.200256:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4879 00000100:00100000:0.0:1767319169.200267:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920545536:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319169.200294:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319169.200366:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920545536:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 100us (359us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319169.200375:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4879 00000100:00100000:1.0:1767319169.201574:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4457 00000100:00100000:1.0:1767319169.201591:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920545664:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319169.201688:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920545664:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 104us (1743us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319169.201696:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4457 00000100:00080000:2.0:1767319169.204479:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319169.243614:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319169.243626:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319169.243629:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319169.243633:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319169.243638:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319169.243659:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4412 00000100:00080000:2.0:1767319169.243666:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767319169.243673:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319169.243716:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405c00 x1853163985100544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319169.243781:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985100544:0@lo:38:kworker.0 00000100:00100000:3.0:1767319169.243847:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319169.243978:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100544 00000100:00100000:1.0:1767319169.243987:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100544 00000100:00100000:1.0:1767319169.244034:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4880 00000100:00100000:1.0:1767319169.244042:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985100544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319169.244079:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319169.244139:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319169.244143:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319169.244145:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319169.244149:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4412 00000100:00080000:2.0:1767319169.244152:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767319169.244156:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319169.244195:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404700 x1853163985100672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319169.244615:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985100672:0@lo:38:kworker.0 00000100:00100000:3.0:1767319169.244656:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319169.244694:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100672 00000100:00100000:0.0:1767319169.244700:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100672 00000100:00100000:0.0:1767319169.244739:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1710 00000100:00100000:0.0:1767319169.244746:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985100672:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319169.244758:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319169.244786:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985100672:12345-0@lo:38:kworker.0 Request processed in 41us (140us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319169.244791:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1710 00000100:00100000:1.0:1767319169.244922:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985100544:12345-0@lo:38:kworker.0 Request processed in 879us (1073us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319169.244930:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4880 00000100:00080000:2.0:1767319169.245011:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319169.245015:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319169.245021:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319169.245025:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4412 00000100:00080000:2.0:1767319169.245028:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767319169.245033:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319169.245048:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319169.245062:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985100800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319169.245062:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407480 x1853163985100928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319169.245097:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319169.245125:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100800 00000100:00100000:1.0:1767319169.245128:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100800 00000100:00100000:1.0:1767319169.245183:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4458 00000100:00100000:1.0:1767319169.245191:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985100800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319169.245235:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985100800:12345-0@lo:400:kworker.0 Request processed in 45us (136us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319169.245241:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4458 00000100:00100000:1.0:1767319169.245278:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985100800:0@lo:400:kworker.0 00000100:00080000:1.0:1767319169.245298:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319169.245395:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319169.245398:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319169.245400:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319169.245404:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4412 00000100:00080000:2.0:1767319169.245407:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767319169.245411:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319169.245424:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405880 x1853163985101184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319169.245438:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985101056:0@lo:400:kworker.0 00000100:00100000:1.0:1767319169.245470:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319169.245485:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985101440:0@lo:400:kworker.0 00000100:00100000:1.0:1767319169.245502:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101056 00000100:00100000:1.0:1767319169.245507:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101056 00000100:00100000:2.0:1767319169.245523:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319169.245544:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4459 00000100:00100000:2.0:1767319169.245550:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319169.245554:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985101056:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319169.245560:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985101312:0@lo:400:kworker.0 00000100:00100000:1.0:1767319169.245593:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985101056:12345-0@lo:400:kworker.0 Request processed in 41us (123us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319169.245599:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4459 00000100:00080000:1.0:1767319169.245612:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319169.245638:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985101056:0@lo:400:kworker.0 00000100:00100000:2.0:1767319169.245697:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319169.245771:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101440 00000100:00100000:0.0:1767319169.245776:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101440 00000100:00100000:0.0:1767319169.245821:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101312 00000100:00100000:0.0:1767319169.245824:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101312 00000100:00100000:0.0:1767319169.245845:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2937 00000100:00100000:0.0:1767319169.245852:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985101440:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319169.245888:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985101440:12345-0@lo:400:kworker.0 Request processed in 36us (366us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319169.245894:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2937 00000100:00100000:0.0:1767319169.245906:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2938 00000100:00100000:0.0:1767319169.245912:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985101312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319169.245934:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985101312:12345-0@lo:400:kworker.0 Request processed in 23us (239us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319169.245940:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2938 00000100:00100000:0.0:1767319169.245973:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985101312:0@lo:400:kworker.0 00000100:00100000:0.0:1767319169.245982:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985101440:0@lo:400:kworker.0 00000100:00080000:2.0:1767319169.246039:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319169.246042:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319169.246390:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319169.246396:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319169.246399:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985100544:0@lo:38:kworker.0 00000100:00100000:3.0:1767319169.246415:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985100928:0@lo:38:kworker.0 00000100:00100000:3.0:1767319169.246440:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319169.246466:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985101184:0@lo:38:kworker.0 00000100:00100000:3.0:1767319169.246484:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319169.246593:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319169.246598:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319169.246601:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985100672:0@lo:38:kworker.0 00000100:00100000:0.0:1767319169.246733:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985100928 00000100:00100000:0.0:1767319169.246738:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985100928 00000100:00100000:0.0:1767319169.246766:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101184 00000100:00100000:0.0:1767319169.246769:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101184 00000100:00100000:0.0:1767319169.246790:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4881 00000100:00100000:0.0:1767319169.246795:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985100928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319169.246809:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319169.246841:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985100928:12345-0@lo:38:kworker.0 Request processed in 45us (400us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319169.246846:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4881 00000100:00100000:0.0:1767319169.246857:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4882 00000100:00100000:0.0:1767319169.246860:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985101184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319169.246869:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319169.246888:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985101184:12345-0@lo:38:kworker.0 Request processed in 27us (403us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319169.246893:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4882 00000100:00080000:3.0:1767319169.246955:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319169.246959:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319169.246962:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985100928:0@lo:38:kworker.0 00000100:00080000:3.0:1767319169.246969:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319169.246971:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319169.246973:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985101184:0@lo:38:kworker.0 00010000:00010000:0.0:1767319169.887580:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319170.907524:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319171.804038:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985101568:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319171.804110:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319171.804264:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101568 00000100:00100000:2.0:1767319171.804273:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101568 00000100:00100000:2.0:1767319171.804432:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2100 00000100:00100000:2.0:1767319171.804444:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985101568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319171.804477:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767319171.804569:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319171.804589:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985101696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319171.804627:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319171.804657:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101696 00000100:00100000:3.0:1767319171.804661:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101696 00000100:00100000:3.0:1767319171.804701:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2101 00000100:00100000:3.0:1767319171.804708:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985101696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319171.804734:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319171.804764:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985101696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 57us (137us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319171.804772:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2101 00000100:00100000:3.0:1767319171.804812:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985101696:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319171.804829:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319171.807934:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985101568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3483us (3813us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319171.807948:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2100 00000100:00100000:0.0:1767319171.809089:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985101568:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319171.932654:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319172.956691:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319173.979844:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319174.315294:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319174.315659:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920545920 00000100:00100000:2.0:1767319174.315671:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920545920 00000100:00100000:2.0:1767319174.315731:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4460 00000100:00100000:2.0:1767319174.315742:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920545920:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319174.315814:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920545920:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (528us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319174.315822:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4460 00000100:00080000:3.0:1767319174.317072:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319174.317147:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319174.317387:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920545792 00000100:00100000:2.0:1767319174.317395:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920545792 00000100:00100000:2.0:1767319174.317489:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4883 00000100:00100000:2.0:1767319174.317496:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920545792:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319174.317518:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319174.317595:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920545792:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 97us (450us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319174.317602:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4883 00000100:00080000:3.0:1767319174.364636:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319174.364646:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319174.364650:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319174.364654:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319174.364657:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319174.364666:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4417 00000100:00080000:3.0:1767319174.364671:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767319174.364677:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319174.364713:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985101824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319174.364739:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319174.364742:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319174.364744:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319174.364747:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4417 00000100:00080000:3.0:1767319174.364749:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767319174.364753:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319174.364766:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6d80 x1853163985101952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319174.364810:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319174.364814:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319174.364816:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319174.364820:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4417 00000100:00080000:3.0:1767319174.364822:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767319174.364826:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319174.365023:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985102080:0@lo:400:kworker.0 00000100:00100000:1.0:1767319174.365090:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319174.365197:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102080 00000100:00100000:1.0:1767319174.365205:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102080 00000100:00100000:1.0:1767319174.365258:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4461 00000100:00100000:1.0:1767319174.365267:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985102080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319174.365319:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985102080:12345-0@lo:400:kworker.0 Request processed in 53us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319174.365326:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4461 00000100:00100000:1.0:1767319174.365370:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985102080:0@lo:400:kworker.0 00000100:00080000:1.0:1767319174.365417:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319174.365636:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985101824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319174.365669:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319174.365700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985101952:0@lo:38:kworker.0 00000100:00100000:0.0:1767319174.365726:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319174.366025:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101952 00000100:00100000:2.0:1767319174.366030:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101952 00000100:00100000:2.0:1767319174.366061:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1711 00000100:00100000:2.0:1767319174.366067:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985101952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319174.366087:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319174.366125:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985101952:12345-0@lo:38:kworker.0 Request processed in 57us (397us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319174.366132:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1711 00000100:00080000:2.0:1767319174.366160:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319174.366191:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319174.366194:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985101952:0@lo:38:kworker.0 00000100:00100000:3.0:1767319174.366328:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6680 x1853163985102208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319174.366375:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319174.366379:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319174.366381:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319174.366384:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4417 00000100:00080000:3.0:1767319174.366387:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767319174.366391:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319174.366405:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6a00 x1853163985102464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319174.366447:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985102592:0@lo:400:kworker.0 00000100:00100000:3.0:1767319174.366497:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319174.366522:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319174.366528:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985102720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319174.366561:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319174.366592:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102592 00000100:00100000:3.0:1767319174.366596:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102592 00000100:00100000:3.0:1767319174.366641:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102720 00000100:00100000:3.0:1767319174.366644:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102720 00000100:00100000:3.0:1767319174.366663:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2939 00000100:00100000:3.0:1767319174.366669:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985102592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319174.366705:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985102592:12345-0@lo:400:kworker.0 Request processed in 36us (208us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319174.366711:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2939 00000100:00100000:3.0:1767319174.366723:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2940 00000100:00100000:3.0:1767319174.366728:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985102720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319174.366753:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985102720:12345-0@lo:400:kworker.0 Request processed in 25us (203us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319174.366758:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2940 00000100:00100000:3.0:1767319174.366793:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985102720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319174.366802:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985102592:0@lo:400:kworker.0 00000100:00080000:3.0:1767319174.366818:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319174.366822:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319174.366829:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985102336:0@lo:400:kworker.0 00000100:00100000:3.0:1767319174.367412:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319174.367563:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985102208:0@lo:38:kworker.0 00000100:00100000:2.0:1767319174.367612:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319174.367635:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985102464:0@lo:38:kworker.0 00000100:00100000:2.0:1767319174.367655:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319174.367865:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985101824 00000100:00100000:1.0:1767319174.367880:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985101824 00000100:00100000:1.0:1767319174.367912:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102336 00000100:00100000:1.0:1767319174.367915:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102336 00000100:00100000:1.0:1767319174.367943:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102208 00000100:00100000:1.0:1767319174.367946:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102208 00000100:00100000:1.0:1767319174.367953:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102464 00000100:00100000:1.0:1767319174.367955:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102464 00000100:00100000:1.0:1767319174.367964:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4462 00000100:00100000:1.0:1767319174.367972:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985102336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319174.368026:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985102336:12345-0@lo:400:kworker.0 Request processed in 54us (615us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319174.368036:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4462 00000100:00100000:1.0:1767319174.368051:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4884 00000100:00100000:1.0:1767319174.368055:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985101824:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319174.368071:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319174.368110:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985101824:12345-0@lo:38:kworker.0 Request processed in 42us (2429us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319174.368116:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4884 00000100:00100000:1.0:1767319174.368128:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4885 00000100:00100000:1.0:1767319174.368131:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985102208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319174.368139:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319174.368161:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985102208:12345-0@lo:38:kworker.0 Request processed in 28us (556us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319174.368201:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4885 00000100:00100000:1.0:1767319174.368213:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4886 00000100:00100000:1.0:1767319174.368217:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985102464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319174.368225:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319174.368250:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985102464:12345-0@lo:38:kworker.0 Request processed in 32us (593us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319174.368256:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4886 00000100:00080000:1.0:1767319174.368274:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319174.368301:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985102336:0@lo:400:kworker.0 00000100:00080000:2.0:1767319174.369090:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319174.369098:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319174.369102:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985102208:0@lo:38:kworker.0 00000100:00080000:2.0:1767319174.369112:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319174.369115:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319174.369118:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985102464:0@lo:38:kworker.0 00000100:00080000:2.0:1767319174.369127:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319174.369131:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319174.369133:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985101824:0@lo:38:kworker.0 00010000:00010000:2.0:1767319175.008489:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319176.027378:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319176.923474:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319176.923499:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985102848:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319176.923595:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319176.923658:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102848 00000100:00100000:2.0:1767319176.923667:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102848 00000100:00100000:2.0:1767319176.923787:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2102 00000100:00100000:2.0:1767319176.923799:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985102848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319176.923831:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319176.923848:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985102976:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319176.923881:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985102848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319176.923889:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319176.923894:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2102 00000100:00100000:3.0:1767319176.923910:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985102976 00000100:00100000:3.0:1767319176.923916:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985102976 00000100:00100000:0.0:1767319176.923937:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985102848:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319176.924545:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319176.924551:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319176.930760:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2103 00000100:00100000:3.0:1767319176.930779:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985102976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319176.930818:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319176.930917:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985102976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 145us (7029us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319176.930925:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2103 00000100:00100000:2.0:1767319176.931071:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985102976:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319177.051371:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319178.075678:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319179.099385:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319179.438361:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319179.438403:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319179.438479:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920546048 00000100:00100000:0.0:1767319179.438496:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920546048 00000100:00100000:0.0:1767319179.438666:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920546176 00000100:00100000:0.0:1767319179.438670:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920546176 00000100:00080000:2.0:1767319179.438767:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319179.441919:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4463 00000100:00100000:1.0:1767319179.441944:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920546176:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319179.442225:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4887 00000100:00100000:3.0:1767319179.442237:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920546048:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767319179.442264:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920546176:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 328us (3854us total) trans 0 rc 0/0 00010000:00020000:3.0:1767319179.442264:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319179.442324:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920546048:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 87us (3977us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319179.442812:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4463 00000100:00100000:3.0:1767319179.443025:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4887 00000100:00080000:0.0:1767319179.485383:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319179.485399:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319179.485403:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319179.485411:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319179.485415:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319179.485425:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4422 00000100:00080000:0.0:1767319179.485431:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319179.485438:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319179.485482:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4a80 x1853163985103104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319179.485513:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319179.485517:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319179.485519:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319179.485522:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4422 00000100:00080000:0.0:1767319179.485525:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319179.485529:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319179.485542:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985103232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319179.485576:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319179.485579:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319179.485581:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319179.485584:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4422 00000100:00080000:0.0:1767319179.485587:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319179.485589:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319179.485600:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163985103488/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319179.485617:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319179.485621:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319179.485623:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319179.485628:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4422 00000100:00080000:0.0:1767319179.485630:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319179.485633:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319179.485662:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985103360:0@lo:400:kworker.0 00000100:00100000:0.0:1767319179.485664:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1f80 x1853163985103744/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319179.485789:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985103616:0@lo:400:kworker.0 00000100:00100000:0.0:1767319179.485844:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319179.485917:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985103616 00000100:00100000:0.0:1767319179.485925:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985103616 00000100:00100000:0.0:1767319179.485967:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4464 00000100:00100000:0.0:1767319179.485975:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985103616:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319179.486023:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985103616:12345-0@lo:400:kworker.0 Request processed in 48us (178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319179.486031:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4464 00000100:00100000:0.0:1767319179.486070:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985103616:0@lo:400:kworker.0 00000100:00100000:1.0:1767319179.486373:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985103104:0@lo:38:kworker.0 00000100:00100000:3.0:1767319179.486424:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319179.486474:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985103360 00000100:00100000:1.0:1767319179.486481:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985103360 00000100:00100000:1.0:1767319179.486521:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4465 00000100:00100000:1.0:1767319179.486530:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985103360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319179.486580:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985103360:12345-0@lo:400:kworker.0 Request processed in 51us (158us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319179.486585:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4465 00000100:00100000:0.0:1767319179.486645:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319179.486675:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985103232:0@lo:38:kworker.0 00000100:00100000:0.0:1767319179.486699:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319179.486721:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985103488:0@lo:38:kworker.0 00000100:00100000:0.0:1767319179.486738:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319179.486752:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985103744:0@lo:38:kworker.0 00000100:00100000:0.0:1767319179.486767:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319179.486797:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985103232 00000100:00100000:0.0:1767319179.486801:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985103232 00000100:00100000:0.0:1767319179.486818:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1712 00000100:00100000:0.0:1767319179.486822:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985103232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319179.486840:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319179.486873:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985103232:12345-0@lo:38:kworker.0 Request processed in 49us (173us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319179.486879:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1712 00000100:00100000:0.0:1767319179.486895:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985103104 00000100:00100000:0.0:1767319179.486898:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985103104 00000100:00100000:0.0:1767319179.486912:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985103488 00000100:00100000:0.0:1767319179.486915:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985103488 00000100:00100000:0.0:1767319179.486923:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985103744 00000100:00100000:0.0:1767319179.486925:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985103744 00000100:00100000:0.0:1767319179.486933:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4888 00000100:00100000:0.0:1767319179.486937:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985103104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319179.486946:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319179.486970:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985103104:12345-0@lo:38:kworker.0 Request processed in 31us (324us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319179.486976:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4888 00000100:00100000:0.0:1767319179.486986:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4889 00000100:00100000:0.0:1767319179.486990:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985103488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319179.486999:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319179.487020:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985103488:12345-0@lo:38:kworker.0 Request processed in 28us (281us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319179.487025:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4889 00000100:00100000:0.0:1767319179.487036:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4890 00000100:00100000:0.0:1767319179.487040:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985103744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319179.487048:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319179.487070:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985103744:12345-0@lo:38:kworker.0 Request processed in 29us (303us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319179.487075:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4890 00000100:00080000:0.0:1767319179.487103:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319179.487108:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319179.487111:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985103104:0@lo:38:kworker.0 00000100:00080000:0.0:1767319179.487118:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319179.487121:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319179.487124:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985103232:0@lo:38:kworker.0 00000100:00080000:0.0:1767319179.487131:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319179.487134:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319179.487136:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985103488:0@lo:38:kworker.0 00000100:00080000:0.0:1767319179.487143:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319179.487146:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319179.487148:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985103744:0@lo:38:kworker.0 00000100:00100000:2.0:1767319179.487535:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985103872:0@lo:400:kworker.0 00000100:00100000:2.0:1767319179.487577:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319179.487621:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319179.487628:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985104000:0@lo:400:kworker.0 00000100:00100000:3.0:1767319179.487645:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985103360:0@lo:400:kworker.0 00000100:00100000:2.0:1767319179.487659:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319179.487717:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319179.487989:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985103872 00000100:00100000:1.0:1767319179.487995:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985103872 00000100:00100000:1.0:1767319179.488043:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104000 00000100:00100000:1.0:1767319179.488047:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104000 00000100:00100000:1.0:1767319179.488067:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2941 00000100:00100000:1.0:1767319179.488074:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985103872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319179.488116:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985103872:12345-0@lo:400:kworker.0 Request processed in 42us (538us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319179.488122:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2941 00000100:00100000:1.0:1767319179.488134:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2942 00000100:00100000:1.0:1767319179.488140:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985104000:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319179.488192:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985104000:12345-0@lo:400:kworker.0 Request processed in 53us (539us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319179.488196:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2942 00000100:00080000:2.0:1767319179.488295:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319179.488297:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319179.488321:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985104000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319179.488330:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985103872:0@lo:400:kworker.0 00010000:00010000:2.0:1767319180.128704:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319181.149479:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319182.044715:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985104256:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319182.044825:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319182.044889:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319182.044897:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985104128:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319182.044920:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319182.044997:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104256 00000100:00100000:1.0:1767319182.045008:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104256 00000100:00100000:1.0:1767319182.045080:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104128 00000100:00100000:1.0:1767319182.045083:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104128 00000100:00100000:1.0:1767319182.045111:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2104 00000100:00100000:1.0:1767319182.045122:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985104256:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319182.045139:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319182.045149:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767319182.045160:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319182.045210:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2105 00000100:00100000:2.0:1767319182.045232:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985104128:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319182.045266:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319182.045320:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985104128:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (398us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319182.045329:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2105 00000100:00100000:1.0:1767319182.045471:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985104256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 349us (643us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319182.045480:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2104 00000100:00100000:0.0:1767319182.046325:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985104128:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319182.046348:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985104256:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319182.176908:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319183.196443:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319184.220984:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319184.556481:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319184.556664:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319184.560816:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920546432 00000100:00100000:1.0:1767319184.560840:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920546432 00000100:00100000:1.0:1767319184.560972:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920546304 00000100:00100000:1.0:1767319184.560976:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920546304 00000100:00100000:1.0:1767319184.561005:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4466 00000100:00100000:1.0:1767319184.561024:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920546432:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319184.561143:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920546432:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 126us (4676us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319184.561154:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4466 00000100:00100000:1.0:1767319184.561197:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4891 00000100:00100000:1.0:1767319184.561202:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920546304:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319184.561228:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319184.561258:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920546304:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (4594us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319184.561264:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4891 00000100:00080000:3.0:1767319184.561469:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319184.603333:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319184.603348:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319184.603351:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319184.603366:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319184.603372:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319184.603382:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4427 00000100:00080000:0.0:1767319184.603388:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319184.603397:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319184.603449:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0000 x1853163985104384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319184.603469:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319184.603473:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319184.603475:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319184.603479:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4427 00000100:00080000:0.0:1767319184.603481:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319184.603485:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319184.603498:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1880 x1853163985104512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319184.603554:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319184.603558:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319184.603560:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319184.603564:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4427 00000100:00080000:0.0:1767319184.603566:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319184.603569:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319184.603583:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2680 x1853163985104768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319184.603604:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319184.603607:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319184.603609:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319184.603612:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4427 00000100:00080000:0.0:1767319184.603616:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319184.603620:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319184.603633:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0a80 x1853163985105024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319184.603638:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985104896:0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.603698:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985105280:0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.603793:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.603871:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319184.603880:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985105152:0@lo:400:kworker.0 00000100:00100000:1.0:1767319184.604046:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985104384:0@lo:38:kworker.0 00000100:00100000:2.0:1767319184.604190:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985104640:0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.605933:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.606086:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985105280 00000100:00100000:0.0:1767319184.606098:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985105280 00000100:00100000:3.0:1767319184.606143:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.606203:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985105152 00000100:00100000:0.0:1767319184.606208:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985105152 00000100:00100000:0.0:1767319184.606262:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2943 00000100:00100000:0.0:1767319184.606273:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985105280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319184.606289:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2944 00000100:00100000:0.0:1767319184.606431:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985105280:12345-0@lo:400:kworker.0 Request processed in 160us (2637us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319184.606438:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2943 00000100:00100000:0.0:1767319184.606501:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985105280:0@lo:400:kworker.0 00000100:00100000:3.0:1767319184.606575:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985105152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.606669:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104896 00000100:00100000:0.0:1767319184.606675:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104896 00000100:00100000:3.0:1767319184.606917:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985105152:12345-0@lo:400:kworker.0 Request processed in 616us (995us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319184.606929:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2944 00000100:00100000:0.0:1767319184.606940:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4467 00000100:00100000:0.0:1767319184.606950:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985104896:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319184.606971:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319184.606979:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319184.606983:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319184.607030:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985105152:0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.609494:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985104896:12345-0@lo:400:kworker.0 Request processed in 2532us (3343us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319184.609512:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4467 00000100:00100000:1.0:1767319184.612088:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319184.612201:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985104512:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.612266:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319184.612304:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985104768:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.612342:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319184.612377:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985105024:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.612403:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319184.612444:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104384 00000100:00100000:1.0:1767319184.612455:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104384 00000100:00100000:1.0:1767319184.612499:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104768 00000100:00100000:1.0:1767319184.612502:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104768 00000100:00100000:1.0:1767319184.612518:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985105024 00000100:00100000:1.0:1767319184.612521:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985105024 00000100:00100000:1.0:1767319184.612556:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4892 00000100:00100000:1.0:1767319184.612563:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985104384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319184.612595:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319184.612650:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985104384:12345-0@lo:38:kworker.0 Request processed in 85us (574us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319184.612658:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4892 00000100:00100000:1.0:1767319184.612677:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4893 00000100:00100000:1.0:1767319184.612682:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985104768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319184.612689:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319184.612718:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985104768:12345-0@lo:38:kworker.0 Request processed in 35us (375us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319184.612722:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4893 00000100:00100000:1.0:1767319184.612729:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4894 00000100:00100000:1.0:1767319184.612732:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985105024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319184.612738:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319184.612750:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985105024:12345-0@lo:38:kworker.0 Request processed in 17us (347us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319184.612754:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4894 00000100:00080000:1.0:1767319184.612798:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319184.612804:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319184.612808:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985105024:0@lo:38:kworker.0 00000100:00080000:1.0:1767319184.612840:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319184.612845:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319184.612848:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985104384:0@lo:38:kworker.0 00000100:00080000:1.0:1767319184.612855:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319184.612857:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319184.612859:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985104768:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.612900:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104512 00000100:00100000:1.0:1767319184.612903:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104512 00000100:00100000:1.0:1767319184.612919:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1713 00000100:00100000:1.0:1767319184.612922:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985104512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319184.612929:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319184.612952:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985104512:12345-0@lo:38:kworker.0 Request processed in 28us (686us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319184.612958:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1713 00000100:00080000:1.0:1767319184.612983:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319184.612987:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319184.612990:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985104512:0@lo:38:kworker.0 00000100:00100000:3.0:1767319184.614524:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985104896:0@lo:400:kworker.0 00000100:00100000:2.0:1767319184.615953:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.616102:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985104640 00000100:00100000:0.0:1767319184.616113:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985104640 00000100:00100000:0.0:1767319184.616220:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4468 00000100:00100000:0.0:1767319184.616232:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985104640:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319184.616289:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319184.616326:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985104640:12345-0@lo:400:kworker.0 Request processed in 96us (383us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319184.616335:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4468 00000100:00100000:2.0:1767319184.616437:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985104640:0@lo:400:kworker.0 00010000:00010000:1.0:1767319185.246576:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319186.267403:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319187.163590:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985105408:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319187.163675:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319187.163717:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319187.163723:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985105536:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319187.163741:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319187.163770:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985105408 00000100:00100000:2.0:1767319187.163776:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985105408 00000100:00100000:2.0:1767319187.163821:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985105536 00000100:00100000:2.0:1767319187.163823:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985105536 00000100:00100000:2.0:1767319187.163840:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2106 00000100:00100000:2.0:1767319187.163848:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985105408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319187.163877:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319187.163911:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985105408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319187.163918:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2106 00000100:00100000:2.0:1767319187.163933:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2107 00000100:00100000:2.0:1767319187.163938:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985105536:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319187.163950:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319187.163968:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985105536:12345-0@lo:13:osp-pre-0-1.0 Request processed in 31us (226us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319187.163973:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2107 00000100:00080000:2.0:1767319187.163986:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319187.163989:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319187.164021:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985105536:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319187.164031:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985105408:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319187.291415:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319188.316102:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319188.655211:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319188.655249:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920546560 offset 0 length 520: 4 00000100:00100000:2.0:1767319188.655273:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319188.655372:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920546816 00000100:00100000:1.0:1767319188.655382:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920546816 00000100:00100000:1.0:1767319188.655456:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920546688 00000100:00100000:1.0:1767319188.655460:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920546688 00000100:00100000:1.0:1767319188.655485:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4469 00000100:00100000:1.0:1767319188.655495:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920546816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319188.655568:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4895 00000100:00100000:1.0:1767319188.655577:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920546816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (411us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319188.655586:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920546688:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767319188.655597:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4469 00010000:00020000:3.0:1767319188.655623:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319188.655688:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920546688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 106us (413us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319188.655696:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4895 00000100:00080000:2.0:1767319188.659560:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319189.339495:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319189.723456:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319164/real 1767319164] req@ffff8ce1b2dc0000 x1853163985099136/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319189 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319189.723503:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b2dc0000 x1853163985099136/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319189 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319189.723530:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319189.723556:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319189.723561:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985099136:0@lo:250:kworker.0 00000100:00080000:0.0:1767319189.724517:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319189.724529:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319189.724533:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319189.724541:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4412 00000100:00080000:0.0:1767319189.724546:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319189.724564:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319189.724600:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3800 x1853163985105664/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319189.724625:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319189.724629:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319189.724631:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319189.724635:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4432 00000100:00080000:0.0:1767319189.724637:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319189.724641:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319189.724662:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2d80 x1853163985105792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319189.724675:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985105664:0@lo:250:kworker.0 00000100:00080000:0.0:1767319189.724679:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319189.724689:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319189.724691:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319189.724695:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4432 00000100:00080000:0.0:1767319189.724698:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319189.724703:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319189.724723:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3100 x1853163985105920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:3.0:1767319189.724747:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985105664 offset 0 length 520: 4 00000100:00080000:0.0:1767319189.724781:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319189.724787:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319189.724789:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319189.724796:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4432 00000100:00080000:0.0:1767319189.724799:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319189.724800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985105792:0@lo:38:kworker.0 00000100:00080000:0.0:1767319189.724811:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319189.724843:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319189.724871:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985105920:0@lo:38:kworker.0 00000100:00100000:0.0:1767319189.724877:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1880 x1853163985106176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319189.724903:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767319189.724908:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319189.724917:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319189.724920:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319189.724926:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4432 00000100:00080000:0.0:1767319189.724929:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319189.724934:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985106176:0@lo:38:kworker.0 00000100:00080000:0.0:1767319189.724941:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319189.724960:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0000 x1853163985106432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319189.724974:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319189.725001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985106432:0@lo:38:kworker.0 00000100:00100000:3.0:1767319189.725030:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319189.725070:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985106304:0@lo:400:kworker.0 00000100:00100000:0.0:1767319189.725100:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985105792 00000100:00100000:3.0:1767319189.725100:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319189.725113:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985105792 00000100:00100000:3.0:1767319189.725127:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319189.725137:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985106048:0@lo:400:kworker.0 00000100:00100000:0.0:1767319189.725150:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106176 00000100:00100000:0.0:1767319189.725156:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106176 00000100:00100000:3.0:1767319189.725196:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319189.725209:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106432 00000100:00100000:0.0:1767319189.725215:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106432 00000100:00100000:3.0:1767319189.725243:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106304 00000100:00100000:0.0:1767319189.725244:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106048 00000100:00100000:3.0:1767319189.725254:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106304 00000100:00100000:0.0:1767319189.725255:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106048 00000100:00100000:3.0:1767319189.725330:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4470 00000100:00100000:1.0:1767319189.725362:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985105920 00000100:00100000:1.0:1767319189.725369:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985105920 00000100:00100000:1.0:1767319189.725400:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1714 00000100:00100000:1.0:1767319189.725407:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985105920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319189.725427:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319189.725506:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985105920:12345-0@lo:38:kworker.0 Request processed in 98us (614us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319189.725514:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1714 00000100:00100000:1.0:1767319189.725541:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4896 00000100:00100000:1.0:1767319189.725556:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985105792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319189.725567:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319189.725595:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985105792:12345-0@lo:38:kworker.0 Request processed in 48us (753us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319189.725600:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4896 00000100:00100000:1.0:1767319189.725617:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4897 00000100:00100000:1.0:1767319189.725620:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985106176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319189.725628:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319189.725649:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985106176:12345-0@lo:38:kworker.0 Request processed in 27us (681us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319189.725655:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4897 00000100:00100000:1.0:1767319189.725666:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4898 00000100:00100000:1.0:1767319189.725669:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985106432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319189.725677:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319189.725700:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985106432:12345-0@lo:38:kworker.0 Request processed in 29us (670us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319189.725705:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4898 00000100:00100000:0.0:1767319189.725724:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4471 00000100:00100000:2.0:1767319189.725784:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985106560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319189.725836:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319189.725866:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319189.725871:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985106688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319189.725892:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319189.725916:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319189.725928:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106560 00000100:00100000:2.0:1767319189.725931:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106560 00000100:00100000:2.0:1767319189.725960:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106688 00000100:00100000:2.0:1767319189.725962:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106688 00000100:00100000:2.0:1767319189.725982:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2945 00000100:00100000:2.0:1767319189.725988:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985106560:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319189.726027:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985106560:12345-0@lo:400:kworker.0 Request processed in 40us (202us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319189.726034:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2945 00000100:00100000:2.0:1767319189.726046:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2946 00000100:00100000:2.0:1767319189.726052:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985106688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319189.726082:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985106688:12345-0@lo:400:kworker.0 Request processed in 31us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319189.726087:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2946 00000100:00080000:2.0:1767319189.726105:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319189.726108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319189.726133:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985106688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319189.726144:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985106560:0@lo:400:kworker.0 00000100:00100000:3.0:1767319189.726835:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985106304:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319189.727292:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985106304:12345-0@lo:400:kworker.0 Request processed in 1950us (2191us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319189.727445:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985106048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319189.727518:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985106048:12345-0@lo:400:kworker.0 Request processed in 1786us (2353us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319189.728246:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4470 00000100:00080000:1.0:1767319189.728249:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319189.728259:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319189.728274:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985106432:0@lo:38:kworker.0 00000100:00080000:1.0:1767319189.728292:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319189.728297:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319189.728300:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985106176:0@lo:38:kworker.0 00000100:00080000:1.0:1767319189.728311:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319189.728316:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319189.728320:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985105792:0@lo:38:kworker.0 00000100:00080000:1.0:1767319189.728342:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319189.728345:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319189.728349:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985105920:0@lo:38:kworker.0 00000100:00100000:3.0:1767319189.728430:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985106048:0@lo:400:kworker.0 00000100:00100000:3.0:1767319189.728457:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985106304:0@lo:400:kworker.0 00000100:00100000:0.0:1767319189.728470:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4471 00010000:00010000:1.0:1767319190.363589:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319191.387765:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319192.284630:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319192.284653:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985106816:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319192.284722:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319192.284810:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106816 00000100:00100000:0.0:1767319192.284819:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106816 00000100:00100000:0.0:1767319192.284894:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2108 00000100:00100000:0.0:1767319192.284903:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985106816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319192.284931:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319192.284970:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985106816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319192.284977:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2108 00000100:00080000:0.0:1767319192.284998:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319192.285192:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985106944:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319192.285232:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319192.285316:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985106816:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319192.287863:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985106944 00000100:00100000:1.0:1767319192.288226:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985106944 00000100:00080000:0.0:1767319192.288317:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319192.288477:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2109 00000100:00100000:1.0:1767319192.288489:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985106944:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319192.288521:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319192.288624:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985106944:12345-0@lo:13:osp-pre-0-1.0 Request processed in 138us (3391us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319192.288632:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2109 00000100:00100000:3.0:1767319192.288693:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985106944:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319192.413759:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319193.440120:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319193.782340:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319193.782384:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319193.782657:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920547072 00000100:00100000:2.0:1767319193.782667:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920547072 00000100:00100000:2.0:1767319193.782736:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920546944 00000100:00100000:2.0:1767319193.782739:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920546944 00000100:00100000:2.0:1767319193.782766:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4472 00000100:00100000:2.0:1767319193.782776:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920547072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319193.782967:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920547072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 192us (635us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319193.782975:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4472 00000100:00100000:2.0:1767319193.782995:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4899 00000100:00100000:2.0:1767319193.782999:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920546944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319193.783018:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319193.783044:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920546944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (660us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319193.783050:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4899 00000100:00080000:3.0:1767319193.784126:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319194.467560:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319194.844129:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319194.844146:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319194.844150:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319194.844155:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319194.844160:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319194.844201:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4437 00000100:00080000:0.0:1767319194.844208:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319194.844217:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319194.844276:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0380 x1853163985107072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319194.844308:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319194.844312:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319194.844324:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319194.844328:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4437 00000100:00080000:0.0:1767319194.844331:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319194.844335:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319194.844349:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3100 x1853163985107200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319194.844402:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319194.844407:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319194.844409:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319194.844413:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4437 00000100:00080000:0.0:1767319194.844416:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319194.844419:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319194.844435:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1880 x1853163985107456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319194.844457:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985107328:0@lo:400:kworker.0 00000100:00080000:0.0:1767319194.844464:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319194.844467:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319194.844468:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319194.844472:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4437 00000100:00080000:0.0:1767319194.844474:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319194.844478:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319194.844490:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0a80 x1853163985107712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319194.844555:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319194.844629:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319194.844636:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985107584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319194.844659:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319194.844702:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107328 00000100:00100000:2.0:1767319194.844711:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107328 00000100:00100000:3.0:1767319194.844729:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985107840:0@lo:400:kworker.0 00000100:00100000:2.0:1767319194.844772:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107584 00000100:00100000:2.0:1767319194.844775:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107584 00000100:00100000:2.0:1767319194.844804:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4473 00000100:00100000:2.0:1767319194.844813:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985107328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319194.846126:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985107072:0@lo:38:kworker.0 00000100:00100000:0.0:1767319194.846288:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4474 00000100:00100000:0.0:1767319194.846304:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985107584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319194.848719:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985107328:12345-0@lo:400:kworker.0 Request processed in 3894us (4152us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319194.848736:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4473 00000100:00080000:2.0:1767319194.848796:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319194.848864:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985107328:0@lo:400:kworker.0 00000100:00100000:2.0:1767319194.848923:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985107968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319194.848971:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319194.849039:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107968 00000100:00100000:2.0:1767319194.849047:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107968 00000100:00100000:3.0:1767319194.849101:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319194.849105:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2947 00000100:00100000:2.0:1767319194.849116:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985107968:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319194.849143:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107840 00000100:00100000:3.0:1767319194.849146:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107840 00000100:00100000:3.0:1767319194.849199:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2948 00000100:00100000:2.0:1767319194.849200:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985107968:12345-0@lo:400:kworker.0 Request processed in 86us (229us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319194.849206:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2947 00000100:00100000:3.0:1767319194.849206:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985107840:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319194.849226:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319194.849229:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319194.849246:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985107968:0@lo:400:kworker.0 00000100:00100000:3.0:1767319194.849295:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985107840:12345-0@lo:400:kworker.0 Request processed in 90us (194us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319194.849296:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985107840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319194.849302:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2948 00000100:00100000:1.0:1767319194.849879:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319194.849912:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985107200:0@lo:38:kworker.0 00000100:00100000:1.0:1767319194.849941:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319194.849977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985107456:0@lo:38:kworker.0 00000100:00100000:1.0:1767319194.850035:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319194.850061:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985107712:0@lo:38:kworker.0 00000100:00100000:1.0:1767319194.850084:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319194.850118:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107072 00000100:00100000:1.0:1767319194.850123:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107072 00000100:00100000:1.0:1767319194.850149:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107456 00000100:00100000:1.0:1767319194.850151:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107456 00000100:00100000:1.0:1767319194.850188:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107712 00000100:00100000:1.0:1767319194.850192:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107712 00000100:00100000:1.0:1767319194.850203:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4900 00000100:00100000:1.0:1767319194.850208:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985107072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319194.850231:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319194.850281:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985107072:12345-0@lo:38:kworker.0 Request processed in 71us (403us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319194.850287:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4900 00000100:00100000:1.0:1767319194.850300:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4901 00000100:00100000:1.0:1767319194.850304:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985107456:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319194.850567:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4902 00000100:00100000:2.0:1767319194.850573:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985107712:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319194.851129:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985107200 00000100:00100000:3.0:1767319194.851139:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985107200 00000100:00100000:3.0:1767319194.851209:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1715 00000100:00100000:3.0:1767319194.851215:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985107200:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319194.851267:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319194.851339:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985107456:12345-0@lo:38:kworker.0 Request processed in 1033us (1303us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319194.851347:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4901 00000100:00080000:1.0:1767319194.851409:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319194.851415:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319194.851419:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985107072:0@lo:38:kworker.0 00000100:00080000:1.0:1767319194.851430:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319194.851433:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319194.851436:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985107456:0@lo:38:kworker.0 00010000:00020000:2.0:1767319194.854717:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319194.854867:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985107712:12345-0@lo:38:kworker.0 Request processed in 4288us (4778us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319194.854883:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4902 00010000:00020000:3.0:1767319194.855009:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319194.855051:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985107200:12345-0@lo:38:kworker.0 Request processed in 3835us (5109us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319194.855060:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1715 00000100:00080000:1.0:1767319194.855089:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319194.855097:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319194.855102:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985107200:0@lo:38:kworker.0 00000100:00080000:1.0:1767319194.855120:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319194.855123:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319194.855128:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985107712:0@lo:38:kworker.0 00000100:00100000:0.0:1767319194.855725:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985107584:12345-0@lo:400:kworker.0 Request processed in 9423us (11060us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319194.855736:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4474 00000100:00100000:2.0:1767319194.855956:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985107584:0@lo:400:kworker.0 00010000:00010000:1.0:1767319195.483783:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319196.507796:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319197.404132:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985108096:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319197.404258:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319197.404616:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108096 00000100:00100000:1.0:1767319197.404629:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108096 00000100:00100000:1.0:1767319197.404729:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2110 00000100:00100000:1.0:1767319197.404741:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985108096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319197.404785:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319197.404853:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985108096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (594us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319197.404942:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2110 00000100:00100000:1.0:1767319197.405005:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985108096:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319197.405042:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319197.405050:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985108224:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319197.405080:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319197.405198:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319197.405283:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108224 00000100:00100000:2.0:1767319197.405287:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108224 00000100:00100000:2.0:1767319197.405336:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2111 00000100:00100000:2.0:1767319197.405343:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985108224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319197.405365:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319197.405557:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985108224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 213us (475us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319197.405565:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2111 00000100:00080000:3.0:1767319197.405622:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319197.405690:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985108224:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319197.532700:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319198.555731:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319198.896261:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319198.896905:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319198.903588:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920547200 00000100:00100000:0.0:1767319198.903604:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920547200 00000100:00100000:0.0:1767319198.903799:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920547328 00000100:00100000:0.0:1767319198.903804:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920547328 00000100:00100000:1.0:1767319198.903845:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4903 00000100:00100000:1.0:1767319198.903855:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920547200:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319198.903892:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319198.903963:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4475 00000100:00100000:1.0:1767319198.903969:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920547200:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 112us (7727us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319198.903976:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920547328:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319198.903984:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4903 00000100:00100000:0.0:1767319198.904023:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920547328:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 51us (7121us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319198.904030:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4475 00000100:00080000:3.0:1767319198.904157:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319199.579595:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319199.963306:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319199.963316:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319199.963321:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319199.963325:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319199.963330:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319199.963338:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4443 00000100:00080000:0.0:1767319199.963343:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319199.963350:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319199.963389:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0a80 x1853163985108352/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319199.963404:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319199.963407:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319199.963409:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319199.963412:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4443 00000100:00100000:1.0:1767319199.963413:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985108352:0@lo:38:kworker.0 00000100:00080000:0.0:1767319199.963414:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319199.963418:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319199.963429:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3100 x1853163985108480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319199.963463:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319199.963466:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319199.963468:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319199.963471:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4443 00000100:00080000:0.0:1767319199.963473:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319199.963475:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319199.963478:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319199.963485:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0380 x1853163985108736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319199.963504:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319199.963506:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319199.963507:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319199.963510:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4443 00000100:00080000:0.0:1767319199.963512:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319199.963514:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319199.963522:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2680 x1853163985108992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319199.963531:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985108480:0@lo:38:kworker.0 00000100:00100000:1.0:1767319199.963555:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319199.963575:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985108736:0@lo:38:kworker.0 00000100:00100000:0.0:1767319199.963583:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985109120:0@lo:400:kworker.0 00000100:00100000:1.0:1767319199.963595:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319199.963612:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319199.963614:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985108992:0@lo:38:kworker.0 00000100:00100000:0.0:1767319199.963628:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319199.963631:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319199.963633:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985109248:0@lo:400:kworker.0 00000100:00100000:0.0:1767319199.963654:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319199.963674:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108352 00000100:00100000:0.0:1767319199.963687:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108736 00000100:00100000:0.0:1767319199.963689:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108736 00000100:00100000:1.0:1767319199.963689:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108352 00000100:00100000:1.0:1767319199.963718:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108992 00000100:00100000:1.0:1767319199.963721:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108992 00000100:00100000:3.0:1767319199.963787:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985108864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319199.963817:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319199.964193:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985108608:0@lo:400:kworker.0 00000100:00100000:3.0:1767319199.972918:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108480 00000100:00100000:3.0:1767319199.972932:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108480 00000100:00100000:3.0:1767319199.972981:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1716 00000100:00100000:3.0:1767319199.972989:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985108480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319199.973015:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319199.973100:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985108480:12345-0@lo:38:kworker.0 Request processed in 109us (9542us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319199.973108:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1716 00000100:00100000:3.0:1767319199.973134:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985109120 00000100:00100000:3.0:1767319199.973139:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985109120 00000100:00100000:3.0:1767319199.973201:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985109248 00000100:00100000:3.0:1767319199.973205:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985109248 00000100:00100000:3.0:1767319199.973228:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2949 00000100:00100000:3.0:1767319199.973236:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985109120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319199.973284:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985109120:12345-0@lo:400:kworker.0 Request processed in 49us (9671us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319199.973290:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2949 00000100:00100000:3.0:1767319199.973302:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2950 00000100:00100000:3.0:1767319199.973309:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a06450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985109248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319199.973330:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a06450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985109248:12345-0@lo:400:kworker.0 Request processed in 23us (9676us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319199.973336:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2950 00000100:00080000:3.0:1767319199.974066:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319199.974080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319199.974126:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985109248:0@lo:400:kworker.0 00000100:00100000:3.0:1767319199.974140:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985109120:0@lo:400:kworker.0 00000100:00080000:3.0:1767319199.975630:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319199.975641:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319199.975646:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985108480:0@lo:38:kworker.0 00000100:00100000:3.0:1767319199.975778:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319199.976043:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108864 00000100:00100000:0.0:1767319199.976053:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108864 00000100:00100000:1.0:1767319199.976067:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4904 00000100:00100000:1.0:1767319199.976087:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985108352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319199.976107:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319199.976121:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4476 00000100:00100000:0.0:1767319199.976131:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985108864:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319199.976192:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985108352:12345-0@lo:38:kworker.0 Request processed in 104us (12711us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319199.976199:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4904 00000100:00100000:0.0:1767319199.976208:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985108864:12345-0@lo:400:kworker.0 Request processed in 78us (12388us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319199.976217:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4476 00000100:00100000:1.0:1767319199.976223:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4905 00000100:00100000:1.0:1767319199.976229:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985108736:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319199.976238:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4906 00010000:00020000:1.0:1767319199.976241:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319199.976246:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985108992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319199.976259:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319199.976285:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985108992:12345-0@lo:38:kworker.0 Request processed in 39us (12652us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319199.976290:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4906 00000100:00080000:0.0:1767319199.976308:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319199.976334:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985108864:0@lo:400:kworker.0 00000100:00100000:1.0:1767319199.976356:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985108736:12345-0@lo:38:kworker.0 Request processed in 125us (12759us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319199.976361:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4905 00000100:00080000:1.0:1767319199.976387:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319199.976390:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319199.976394:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985108992:0@lo:38:kworker.0 00000100:00080000:1.0:1767319199.976401:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319199.976403:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319199.976405:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985108736:0@lo:38:kworker.0 00000100:00080000:1.0:1767319199.976411:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319199.976413:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319199.976414:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985108352:0@lo:38:kworker.0 00000100:00100000:2.0:1767319199.986323:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985108608 00000100:00100000:2.0:1767319199.986339:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985108608 00000100:00100000:2.0:1767319199.986424:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4477 00000100:00100000:2.0:1767319199.986434:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985108608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319199.986512:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985108608:12345-0@lo:400:kworker.0 Request processed in 79us (10734us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319199.986520:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4477 00000100:00100000:3.0:1767319199.986594:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985108608:0@lo:400:kworker.0 00000100:00080000:0.0:1767319199.986859:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319200.607466:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319201.628775:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319202.523631:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985109504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319202.523699:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319202.523749:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319202.523755:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985109376:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319202.523777:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319202.525840:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985109504 00000100:00100000:2.0:1767319202.525854:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985109504 00000100:00100000:2.0:1767319202.525939:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985109376 00000100:00100000:2.0:1767319202.525943:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985109376 00000100:00100000:2.0:1767319202.525976:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2112 00000100:00100000:2.0:1767319202.525987:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985109504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319202.526019:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319202.526101:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985109504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (2401us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319202.526109:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2112 00000100:00080000:3.0:1767319202.526246:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319202.526250:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319202.526276:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2113 00000100:00100000:3.0:1767319202.526286:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985109376:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319202.526302:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319202.526328:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985109376:12345-0@lo:13:osp-pre-0-1.0 Request processed in 46us (2552us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319202.526332:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2113 00000100:00100000:1.0:1767319202.526404:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985109376:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319202.526415:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985109504:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319202.651475:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319203.676625:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319204.009603:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319204.009686:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920547584 00000100:00100000:1.0:1767319204.009697:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920547584 00000100:00080000:3.0:1767319204.009771:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319204.009777:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4478 00000100:00100000:1.0:1767319204.009789:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920547584:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319204.009867:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920547584:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (274us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319204.009875:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4478 00000100:00100000:1.0:1767319204.011332:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319204.011416:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920547456 00000100:00100000:2.0:1767319204.011425:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920547456 00000100:00100000:2.0:1767319204.011468:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4907 00000100:00100000:2.0:1767319204.011474:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920547456:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319204.011494:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319204.011550:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920547456:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 74us (226us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319204.011557:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4907 00010000:00010000:2.0:1767319204.699483:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319205.083299:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319205.083310:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319205.083313:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319205.083317:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319205.083322:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319205.083330:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4448 00000100:00080000:3.0:1767319205.083335:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319205.083342:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319205.083382:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4700 x1853163985109632/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319205.083410:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319205.083414:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319205.083416:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319205.083420:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4448 00000100:00080000:3.0:1767319205.083422:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319205.083426:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319205.083438:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6680 x1853163985109760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319205.083485:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319205.083488:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319205.083490:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319205.083494:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4448 00000100:00080000:3.0:1767319205.083496:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319205.083500:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319205.083512:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6d80 x1853163985110016/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319205.083514:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319205.083531:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985109888:0@lo:400:kworker.0 00000100:00080000:3.0:1767319205.083534:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319205.083542:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319205.083545:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319205.083549:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4448 00000100:00080000:3.0:1767319205.083552:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319205.083555:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319205.083579:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc55500 x1853163985110272/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319205.083618:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319205.083647:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985110400:0@lo:400:kworker.0 00000100:00100000:3.0:1767319205.083681:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319205.083741:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985110528:0@lo:400:kworker.0 00000100:00100000:1.0:1767319205.083771:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319205.083837:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319205.083848:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985110144:0@lo:400:kworker.0 00000100:00100000:0.0:1767319205.083882:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319205.083900:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110400 00000100:00100000:2.0:1767319205.083909:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110400 00000100:00100000:0.0:1767319205.083918:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110528 00000100:00100000:0.0:1767319205.083924:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110528 00000100:00100000:2.0:1767319205.084187:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2951 00000100:00100000:3.0:1767319205.084220:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985109888 00000100:00100000:3.0:1767319205.084225:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985109888 00000100:00100000:3.0:1767319205.084257:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110144 00000100:00100000:3.0:1767319205.084260:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110144 00000100:00100000:3.0:1767319205.084541:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4479 00000100:00100000:3.0:1767319205.084551:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985109888:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319205.084554:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985110400:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319205.084607:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985110400:12345-0@lo:400:kworker.0 Request processed in 411us (926us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319205.084612:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2951 00000100:00100000:3.0:1767319205.084626:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985109888:12345-0@lo:400:kworker.0 Request processed in 75us (1009us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319205.084630:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2952 00000100:00100000:2.0:1767319205.084641:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985110528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319205.084641:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4479 00000100:00100000:3.0:1767319205.084668:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4480 00000100:00100000:2.0:1767319205.084670:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985110528:12345-0@lo:400:kworker.0 Request processed in 33us (898us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319205.084676:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2952 00000100:00100000:3.0:1767319205.084680:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985110144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319205.084740:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985110144:12345-0@lo:400:kworker.0 Request processed in 63us (857us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319205.084746:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4480 00000100:00100000:3.0:1767319205.084786:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985110144:0@lo:400:kworker.0 00000100:00100000:1.0:1767319205.093425:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985110528:0@lo:400:kworker.0 00000100:00100000:1.0:1767319205.093486:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985109632:0@lo:38:kworker.0 00000100:00100000:1.0:1767319205.093550:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319205.093609:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985109760:0@lo:38:kworker.0 00000100:00100000:1.0:1767319205.093637:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319205.093661:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985110016:0@lo:38:kworker.0 00000100:00100000:1.0:1767319205.093682:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319205.093700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985110272:0@lo:38:kworker.0 00000100:00100000:1.0:1767319205.093722:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319205.093755:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985109760 00000100:00100000:1.0:1767319205.093760:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985109760 00000100:00100000:1.0:1767319205.093795:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1717 00000100:00100000:1.0:1767319205.093802:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985109760:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319205.093946:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319205.094003:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985109760:12345-0@lo:38:kworker.0 Request processed in 200us (365us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319205.094011:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1717 00000100:00100000:1.0:1767319205.094034:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985109632 00000100:00100000:1.0:1767319205.094038:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985109632 00000100:00100000:1.0:1767319205.094138:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110016 00000100:00100000:1.0:1767319205.094142:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110016 00000100:00100000:1.0:1767319205.094152:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110272 00000100:00100000:1.0:1767319205.094155:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110272 00000100:00100000:1.0:1767319205.094182:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4908 00000100:00100000:1.0:1767319205.094186:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985109632:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319205.094196:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319205.094223:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985109632:12345-0@lo:38:kworker.0 Request processed in 35us (673us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319205.094228:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4908 00000100:00100000:1.0:1767319205.094239:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4909 00000100:00100000:1.0:1767319205.094242:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985110016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319205.094250:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319205.094269:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985110016:12345-0@lo:38:kworker.0 Request processed in 25us (586us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319205.094273:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4909 00000100:00100000:1.0:1767319205.094283:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4910 00000100:00100000:1.0:1767319205.094287:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985110272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319205.094295:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319205.094314:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985110272:12345-0@lo:38:kworker.0 Request processed in 26us (591us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319205.094319:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4910 00000100:00080000:1.0:1767319205.094347:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319205.094353:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319205.094356:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985109632:0@lo:38:kworker.0 00000100:00080000:1.0:1767319205.094364:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319205.094367:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319205.094369:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985109760:0@lo:38:kworker.0 00000100:00080000:1.0:1767319205.094378:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319205.094381:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319205.094383:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985110016:0@lo:38:kworker.0 00000100:00080000:1.0:1767319205.094390:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319205.094393:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319205.094395:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985110272:0@lo:38:kworker.0 00000100:00100000:1.0:1767319205.094494:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985110400:0@lo:400:kworker.0 00000100:00100000:1.0:1767319205.094520:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985109888:0@lo:400:kworker.0 00000100:00080000:1.0:1767319205.094544:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319205.094549:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319205.094551:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319205.724841:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319206.749333:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319207.643774:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985110656:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319207.643860:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319207.643960:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110656 00000100:00100000:2.0:1767319207.643969:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110656 00000100:00100000:2.0:1767319207.644043:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2114 00000100:00100000:2.0:1767319207.644052:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985110656:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319207.644081:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319207.644126:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985110656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (265us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319207.644132:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2114 00000100:00100000:2.0:1767319207.644202:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985110656:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319207.644518:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319207.644715:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319207.650505:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985110784:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319207.650591:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319207.661133:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110784 00000100:00100000:0.0:1767319207.661149:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110784 00000100:00100000:0.0:1767319207.661260:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2115 00000100:00100000:0.0:1767319207.661271:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985110784:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319207.661303:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319207.661640:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985110784:12345-0@lo:13:osp-pre-1-1.0 Request processed in 370us (11048us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319207.661648:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2115 00000100:00080000:3.0:1767319207.661805:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319207.662471:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985110784:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319207.771392:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319208.795393:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319209.130128:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319209.132253:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319209.146184:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920547840 00000100:00100000:1.0:1767319209.146207:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920547840 00000100:00100000:1.0:1767319209.146302:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920547712 00000100:00100000:1.0:1767319209.146306:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920547712 00000100:00100000:1.0:1767319209.146333:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4481 00000100:00100000:1.0:1767319209.146343:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920547840:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319209.146428:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920547840:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (16311us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319209.146436:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4481 00000100:00100000:1.0:1767319209.146456:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4911 00000100:00100000:1.0:1767319209.146460:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920547712:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319209.146479:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319209.146504:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920547712:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (14259us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319209.146510:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4911 00000100:00080000:3.0:1767319209.147457:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319209.819318:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319210.207138:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319210.207149:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319210.207153:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319210.207157:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319210.207192:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319210.207201:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4453 00000100:00080000:2.0:1767319210.207207:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319210.207214:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319210.207253:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6d80 x1853163985110912/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319210.207269:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319210.207273:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319210.207274:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319210.207278:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4453 00000100:00080000:2.0:1767319210.207280:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319210.207283:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319210.207296:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6680 x1853163985111040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319210.207336:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319210.207340:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319210.207342:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319210.207345:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4453 00000100:00080000:2.0:1767319210.207348:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319210.207351:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319210.207363:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7b80 x1853163985111296/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319210.207378:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319210.207381:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319210.207383:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319210.207386:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4453 00000100:00080000:2.0:1767319210.207389:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319210.207392:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319210.207400:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7100 x1853163985111552/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319210.207462:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985111680:0@lo:400:kworker.0 00000100:00100000:2.0:1767319210.207523:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319210.207655:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985111424:0@lo:400:kworker.0 00000100:00100000:3.0:1767319210.207722:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985111168:0@lo:400:kworker.0 00000100:00100000:0.0:1767319210.207789:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319210.207848:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111680 00000100:00100000:0.0:1767319210.207855:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111680 00000100:00100000:0.0:1767319210.207917:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2953 00000100:00100000:0.0:1767319210.207926:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985111680:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319210.207974:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985111680:12345-0@lo:400:kworker.0 Request processed in 49us (450us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319210.207981:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2953 00000100:00080000:0.0:1767319210.208003:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319210.208017:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111424 00000100:00100000:0.0:1767319210.208021:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111424 00000100:00100000:0.0:1767319210.208052:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4482 00000100:00100000:0.0:1767319210.208058:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985111424:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319210.208089:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985111424:12345-0@lo:400:kworker.0 Request processed in 31us (300us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319210.208095:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4482 00000100:00080000:0.0:1767319210.208108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319210.208132:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985111424:0@lo:400:kworker.0 00000100:00100000:0.0:1767319210.208183:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985111808:0@lo:400:kworker.0 00000100:00100000:0.0:1767319210.208208:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319210.208232:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111808 00000100:00100000:0.0:1767319210.208236:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111808 00000100:00100000:0.0:1767319210.208258:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2954 00000100:00100000:0.0:1767319210.208264:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985111808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319210.208296:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985111808:12345-0@lo:400:kworker.0 Request processed in 32us (88us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319210.208302:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2954 00000100:00080000:0.0:1767319210.208313:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319210.208326:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985111808:0@lo:400:kworker.0 00000100:00100000:1.0:1767319210.208417:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985110912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319210.208449:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319210.208473:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985111040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319210.208494:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319210.208516:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111040 00000100:00100000:1.0:1767319210.208520:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111040 00000100:00100000:1.0:1767319210.208541:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1718 00000100:00100000:1.0:1767319210.208545:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985111040:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319210.208563:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319210.208590:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985111040:12345-0@lo:38:kworker.0 Request processed in 43us (94us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319210.208596:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1718 00000100:00080000:1.0:1767319210.208637:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319210.208641:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319210.208645:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985111040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319210.208652:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985111296:0@lo:38:kworker.0 00000100:00100000:1.0:1767319210.208673:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319210.208687:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985111552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319210.208706:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319210.208726:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985110912 00000100:00100000:1.0:1767319210.208730:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985110912 00000100:00100000:1.0:1767319210.208746:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111296 00000100:00100000:1.0:1767319210.208748:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111296 00000100:00100000:1.0:1767319210.208759:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111552 00000100:00100000:1.0:1767319210.208762:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111552 00000100:00100000:1.0:1767319210.208774:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4912 00000100:00100000:1.0:1767319210.208779:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985110912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319210.208790:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319210.208813:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985110912:12345-0@lo:38:kworker.0 Request processed in 32us (363us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319210.208819:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4912 00000100:00100000:1.0:1767319210.208829:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4913 00000100:00100000:1.0:1767319210.208833:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985111296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319210.208842:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319210.208860:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985111296:12345-0@lo:38:kworker.0 Request processed in 26us (187us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319210.208866:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4913 00000100:00100000:1.0:1767319210.208875:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4914 00000100:00100000:1.0:1767319210.208892:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985111552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319210.208902:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319210.208926:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985111552:12345-0@lo:38:kworker.0 Request processed in 44us (218us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319210.208931:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4914 00000100:00080000:1.0:1767319210.208963:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319210.208966:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319210.208969:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985111552:0@lo:38:kworker.0 00000100:00080000:1.0:1767319210.208988:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319210.208992:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319210.208994:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985110912:0@lo:38:kworker.0 00000100:00080000:1.0:1767319210.209002:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319210.209005:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319210.209007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985111296:0@lo:38:kworker.0 00000100:00100000:3.0:1767319210.209097:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319210.209181:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111168 00000100:00100000:1.0:1767319210.209186:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111168 00000100:00100000:1.0:1767319210.209225:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4483 00000100:00100000:1.0:1767319210.209231:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985111168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319210.209264:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985111168:12345-0@lo:400:kworker.0 Request processed in 33us (167us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319210.209271:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4483 00000100:00080000:3.0:1767319210.209325:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319210.209346:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985111168:0@lo:400:kworker.0 00000100:00100000:2.0:1767319210.209425:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985111680:0@lo:400:kworker.0 00010000:00010000:2.0:1767319210.845201:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319211.867422:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319212.763720:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985112064:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319212.763796:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319212.763857:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319212.763864:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985111936:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319212.763886:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319212.764094:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985112064 00000100:00100000:2.0:1767319212.764104:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985112064 00000100:00100000:2.0:1767319212.764185:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985111936 00000100:00100000:2.0:1767319212.764188:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985111936 00000100:00100000:2.0:1767319212.764210:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2116 00000100:00100000:2.0:1767319212.764219:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985112064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319212.764249:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319212.764292:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985112064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (495us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319212.764299:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2116 00000100:00100000:2.0:1767319212.764315:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2117 00000100:00100000:2.0:1767319212.764321:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985111936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319212.764343:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319212.764367:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985111936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (480us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319212.764373:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2117 00000100:00080000:3.0:1767319212.764647:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319212.764652:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319212.765002:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985111936:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319212.765019:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985112064:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319212.893420:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319213.915451:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319214.249538:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319214.249640:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548224 00000100:00100000:0.0:1767319214.249651:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548224 00000100:00100000:0.0:1767319214.249722:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4484 00000100:00100000:0.0:1767319214.249733:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920548224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319214.249829:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920548224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 97us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319214.249837:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4484 00000100:00080000:3.0:1767319214.250378:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:1.0:1767319214.250443:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920547968 offset 0 length 520: 4 00000100:00100000:1.0:1767319214.250468:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319214.250523:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548096 00000100:00100000:2.0:1767319214.250528:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548096 00000100:00100000:2.0:1767319214.250556:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4915 00000100:00100000:2.0:1767319214.250562:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920548096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319214.250582:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319214.250619:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920548096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (150us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319214.250625:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4915 00000100:00100000:1.0:1767319214.299304:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319189/real 1767319189] req@ffff8ce1b34e3800 x1853163985105664/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319214 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319214.299338:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b34e3800 x1853163985105664/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319214 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319214.299351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319214.299380:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319214.299384:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985105664:0@lo:250:kworker.0 00000100:00080000:2.0:1767319214.299462:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319214.299467:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319214.299469:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319214.299474:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4437 00000100:00080000:2.0:1767319214.299478:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319214.299483:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319214.299515:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ce00 x1853163985112192/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319214.299539:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319214.299543:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319214.299545:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319214.299548:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4458 00000100:00080000:2.0:1767319214.299551:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319214.299555:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319214.299569:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985112320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319214.299577:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319214.299580:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319214.299581:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319214.299584:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4458 00000100:00080000:2.0:1767319214.299586:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319214.299590:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319214.299603:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61c00 x1853163985112448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319214.299643:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319214.299647:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319214.299649:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319214.299652:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4458 00000100:00080000:2.0:1767319214.299655:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319214.299658:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319214.299670:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60380 x1853163985112704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319214.299677:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985112576:0@lo:400:kworker.0 00000100:00080000:2.0:1767319214.299689:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319214.299693:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319214.299695:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319214.299699:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4458 00000100:00080000:2.0:1767319214.299702:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319214.299706:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319214.299719:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9180 x1853163985112960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319214.299742:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319214.299797:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985113088:0@lo:400:kworker.0 00000100:00100000:3.0:1767319214.299799:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319214.299818:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985112832:0@lo:400:kworker.0 00000100:00100000:2.0:1767319214.299844:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319214.299852:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319214.299870:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319214.299879:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985113216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319214.299899:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985112576 00000100:00100000:3.0:1767319214.299907:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985112576 00000100:00100000:2.0:1767319214.299921:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319214.299958:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113088 00000100:00100000:3.0:1767319214.299960:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985112832 00000100:00100000:2.0:1767319214.299964:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113088 00000100:00100000:3.0:1767319214.299966:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985112832 00000100:00100000:3.0:1767319214.299997:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4485 00000100:00100000:2.0:1767319214.300007:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113216 00000100:00100000:3.0:1767319214.300007:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985112576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319214.300013:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113216 00000100:00100000:2.0:1767319214.300043:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2955 00000100:00100000:2.0:1767319214.300052:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985113088:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319214.300065:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985112576:12345-0@lo:400:kworker.0 Request processed in 59us (322us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319214.300073:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4485 00000100:00100000:2.0:1767319214.300095:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985113088:12345-0@lo:400:kworker.0 Request processed in 44us (250us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319214.300101:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2955 00000100:00100000:3.0:1767319214.300106:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4486 00000100:00100000:3.0:1767319214.300116:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985112832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319214.300121:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2956 00000100:00100000:2.0:1767319214.300131:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985113216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319214.300149:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985112832:12345-0@lo:400:kworker.0 Request processed in 34us (296us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319214.300155:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4486 00000100:00100000:2.0:1767319214.300197:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985113216:12345-0@lo:400:kworker.0 Request processed in 67us (287us total) trans 0 rc 0/0 00000100:00080000:3.0:1767319214.300201:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319214.300206:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2956 00000100:00080000:3.0:1767319214.300207:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319214.300212:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319214.300240:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985112832:0@lo:400:kworker.0 00000100:00100000:2.0:1767319214.300242:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985113216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319214.300261:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985112576:0@lo:400:kworker.0 00000100:00100000:2.0:1767319214.300264:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985113088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319214.300721:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985112192:0@lo:250:kworker.0 00000400:00000100:1.0:1767319214.300748:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985112192 offset 0 length 520: 4 00000100:00100000:1.0:1767319214.300780:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985112320:0@lo:38:kworker.0 00000100:00100000:1.0:1767319214.300812:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319214.300834:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985112448:0@lo:38:kworker.0 00000100:00100000:1.0:1767319214.300856:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319214.300878:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985112704:0@lo:38:kworker.0 00000100:00100000:1.0:1767319214.300897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319214.300928:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985112960:0@lo:38:kworker.0 00000100:00100000:1.0:1767319214.300947:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319214.300975:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985112320 00000100:00100000:1.0:1767319214.300980:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985112320 00000100:00100000:1.0:1767319214.300999:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985112704 00000100:00100000:1.0:1767319214.301002:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985112704 00000100:00100000:1.0:1767319214.301014:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985112960 00000100:00100000:1.0:1767319214.301016:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985112960 00000100:00100000:1.0:1767319214.301028:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4916 00000100:00100000:1.0:1767319214.301032:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985112320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319214.301049:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319214.301080:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985112320:12345-0@lo:38:kworker.0 Request processed in 46us (276us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319214.301086:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4916 00000100:00100000:1.0:1767319214.301098:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4917 00000100:00100000:1.0:1767319214.301102:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985112704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319214.301111:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319214.301132:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985112704:12345-0@lo:38:kworker.0 Request processed in 28us (233us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319214.301138:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4917 00000100:00100000:1.0:1767319214.301148:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4918 00000100:00100000:1.0:1767319214.301152:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985112960:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319214.301160:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319214.301202:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985112960:12345-0@lo:38:kworker.0 Request processed in 48us (253us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319214.301207:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4918 00000100:00080000:1.0:1767319214.301236:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319214.301240:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319214.301243:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985112320:0@lo:38:kworker.0 00000100:00080000:1.0:1767319214.301251:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319214.301254:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319214.301257:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985112704:0@lo:38:kworker.0 00000100:00080000:1.0:1767319214.301264:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319214.301266:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319214.301269:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985112960:0@lo:38:kworker.0 00000100:00100000:2.0:1767319214.301556:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985112448 00000100:00100000:2.0:1767319214.301561:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985112448 00000100:00100000:2.0:1767319214.301588:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1719 00000100:00100000:2.0:1767319214.301593:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985112448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319214.301603:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319214.302341:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985112448:12345-0@lo:38:kworker.0 Request processed in 744us (1481us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319214.302349:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1719 00000100:00080000:0.0:1767319214.305054:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319214.305518:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319214.305524:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985112448:0@lo:38:kworker.0 00010000:00010000:1.0:1767319214.939424:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319215.963523:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319216.988070:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319217.883371:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985113344:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319217.883442:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319217.883551:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113344 00000100:00100000:2.0:1767319217.883559:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113344 00000100:00100000:0.0:1767319217.883577:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985113472:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319217.883604:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319217.883641:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113472 00000100:00100000:0.0:1767319217.883645:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113472 00000100:00100000:0.0:1767319217.883698:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2118 00000100:00100000:0.0:1767319217.883706:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985113472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319217.883733:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319217.883803:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985113472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (197us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319217.883811:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2118 00000100:00100000:2.0:1767319217.886862:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2119 00000100:00100000:2.0:1767319217.886884:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985113344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319217.886916:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319217.887847:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985113472:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319217.887894:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319217.887899:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319217.896815:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985113344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 9931us (13361us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319217.896831:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2119 00000100:00100000:3.0:1767319217.899367:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985113344:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319218.011816:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319219.035775:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319219.371577:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319219.371775:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548480 00000100:00100000:0.0:1767319219.371786:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548480 00000100:00100000:0.0:1767319219.371952:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4487 00000100:00100000:0.0:1767319219.371962:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920548480:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319219.372066:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920548480:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (495us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319219.372074:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4487 00000100:00080000:1.0:1767319219.374697:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319219.375506:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319219.376531:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548352 00000100:00100000:3.0:1767319219.376541:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548352 00000100:00100000:3.0:1767319219.376589:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4919 00000100:00100000:3.0:1767319219.376595:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920548352:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319219.376619:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319219.376694:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920548352:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 96us (1194us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319219.376701:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4919 00000100:00080000:3.0:1767319219.419610:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.419619:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319219.419628:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.419632:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.419636:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.419644:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4462 00000100:00080000:3.0:1767319219.419648:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.419654:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319219.419689:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60a80 x1853163985113600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319219.419722:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.419727:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.419729:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.419732:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4462 00000100:00080000:3.0:1767319219.419734:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.419737:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319219.419748:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60e00 x1853163985113728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319219.419808:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985113856:0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.419869:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319219.419960:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.419964:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.419966:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.419970:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4462 00000100:00080000:3.0:1767319219.419972:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.419975:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319219.419989:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6adaa00 x1853163985113984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319219.420046:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985113600:0@lo:38:kworker.0 00000100:00100000:0.0:1767319219.420075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319219.420094:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985113728:0@lo:38:kworker.0 00000100:00100000:0.0:1767319219.420116:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319219.420137:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6adaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985113984:0@lo:38:kworker.0 00000100:00100000:0.0:1767319219.420156:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319219.420206:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113728 00000100:00100000:0.0:1767319219.420213:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113728 00000100:00100000:0.0:1767319219.420246:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1720 00000100:00100000:2.0:1767319219.420246:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319219.420252:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985114112:0@lo:400:kworker.0 00000100:00100000:0.0:1767319219.420253:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985113728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319219.420274:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319219.420276:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319219.420310:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985113728:12345-0@lo:38:kworker.0 Request processed in 56us (193us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319219.420318:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1720 00000100:00100000:2.0:1767319219.420337:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113600 00000100:00100000:2.0:1767319219.420341:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113600 00000100:00100000:2.0:1767319219.420359:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113984 00000100:00100000:2.0:1767319219.420361:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113984 00000100:00080000:0.0:1767319219.420363:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319219.420368:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319219.420372:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985113728:0@lo:38:kworker.0 00000100:00100000:2.0:1767319219.420372:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985114112 00000100:00100000:2.0:1767319219.420374:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985114112 00000100:00100000:2.0:1767319219.420415:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4488 00000100:00100000:2.0:1767319219.420421:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985114112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.420461:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985114112:12345-0@lo:400:kworker.0 Request processed in 40us (184us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319219.420468:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4488 00000100:00100000:2.0:1767319219.420481:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4920 00000100:00100000:1.0:1767319219.420483:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985113856 00000100:00100000:2.0:1767319219.420485:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985113600:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319219.420488:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985113856 00010000:00020000:2.0:1767319219.420495:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319219.420510:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4489 00000100:00100000:2.0:1767319219.420516:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985113600:12345-0@lo:38:kworker.0 Request processed in 29us (440us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319219.420517:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985113856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.420522:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4920 00000100:00100000:2.0:1767319219.420532:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4921 00000100:00100000:2.0:1767319219.420536:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985113984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319219.420544:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319219.420545:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985113856:12345-0@lo:400:kworker.0 Request processed in 29us (676us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319219.420550:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4489 00000100:00080000:1.0:1767319219.420574:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319219.420747:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985113984:12345-0@lo:38:kworker.0 Request processed in 208us (589us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319219.420753:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4921 00000100:00100000:2.0:1767319219.420777:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985114112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.420784:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985113856:0@lo:400:kworker.0 00000100:00080000:3.0:1767319219.421293:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.421298:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.421301:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.421304:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4462 00000100:00080000:3.0:1767319219.421307:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.421311:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319219.421336:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6ad9f80 x1853163985114240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319219.421411:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985114368:0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.421446:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319219.421468:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319219.421474:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985114496:0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.421493:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319219.421523:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985114368 00000100:00100000:2.0:1767319219.421528:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985114368 00000100:00080000:2.0:1767319219.421545:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319219.421578:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985114496 00000100:00100000:0.0:1767319219.421582:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985114496 00000100:00100000:2.0:1767319219.421582:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2957 00000100:00100000:2.0:1767319219.421589:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985114368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319219.421608:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2958 00000100:00100000:0.0:1767319219.421614:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985114496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319219.422192:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985114240:0@lo:38:kworker.0 00000100:00080000:3.0:1767319219.422832:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319219.427073:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985114368:12345-0@lo:400:kworker.0 Request processed in 5473us (5615us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319219.427088:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2957 00000100:00100000:1.0:1767319219.427320:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319219.427699:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319219.427705:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319219.427708:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985113600:0@lo:38:kworker.0 00000100:00080000:1.0:1767319219.427733:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319219.427736:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319219.427738:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6adaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985113984:0@lo:38:kworker.0 00000100:00100000:3.0:1767319219.427899:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985114368:0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.427952:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985114240 00000100:00100000:2.0:1767319219.427959:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985114240 00000100:00100000:2.0:1767319219.428035:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4922 00000100:00100000:2.0:1767319219.428043:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985114240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319219.428064:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319219.428127:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985114240:12345-0@lo:38:kworker.0 Request processed in 83us (807us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319219.428134:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4922 00000100:00080000:3.0:1767319219.428198:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319219.428203:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319219.428206:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985114240:0@lo:38:kworker.0 00000100:00100000:0.0:1767319219.428537:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985114496:12345-0@lo:400:kworker.0 Request processed in 6921us (7041us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319219.428545:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2958 00000100:00100000:3.0:1767319219.428619:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985114496:0@lo:400:kworker.0 00010000:00010000:1.0:1767319220.061012:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319221.085162:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319222.107823:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319223.003481:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319223.003785:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319223.003806:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985114752:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319223.003871:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319223.003981:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985114752 00000100:00100000:1.0:1767319223.003989:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985114752 00000100:00100000:1.0:1767319223.004055:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2120 00000100:00100000:1.0:1767319223.004066:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985114752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319223.004094:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319223.004135:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985114752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319223.004143:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2120 00000100:00080000:2.0:1767319223.004231:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319223.004275:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ada300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985114752:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319223.004977:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985114624:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319223.005069:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319223.005202:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985114624 00000100:00100000:1.0:1767319223.005208:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985114624 00000100:00100000:1.0:1767319223.005307:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2121 00000100:00100000:1.0:1767319223.005315:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985114624:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319223.005336:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319223.005375:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985114624:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319223.005382:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2121 00000100:00080000:2.0:1767319223.005855:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319223.011397:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985114624:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319223.132295:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319224.162084:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319224.490789:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319224.490835:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319224.490930:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548608 00000100:00100000:1.0:1767319224.490940:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548608 00000100:00100000:1.0:1767319224.490984:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548736 00000100:00100000:1.0:1767319224.490987:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548736 00000100:00100000:1.0:1767319224.491036:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4490 00000100:00100000:1.0:1767319224.491046:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920548736:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319224.491115:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920548736:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (280us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319224.491123:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4490 00000100:00100000:1.0:1767319224.491142:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4923 00000100:00100000:1.0:1767319224.491147:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920548608:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319224.491208:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319224.491233:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920548608:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 84us (455us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319224.491239:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4923 00000100:00080000:3.0:1767319224.491510:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319224.541075:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319224.541090:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319224.541094:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319224.541099:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319224.541105:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319224.541117:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4467 00000100:00080000:1.0:1767319224.541123:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319224.541131:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319224.541216:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f100 x1853163985114880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319224.541259:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319224.541263:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319224.541265:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319224.541269:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4467 00000100:00080000:1.0:1767319224.541272:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319224.541275:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319224.541292:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985115008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319224.541345:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319224.541351:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319224.541352:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319224.541357:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4467 00000100:00080000:1.0:1767319224.541359:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319224.541363:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319224.541383:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0700 x1853163985115264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319224.541390:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1767319224.541408:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319224.541412:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319224.541415:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319224.541418:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4467 00000100:00080000:1.0:1767319224.541421:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319224.541428:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319224.541431:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985115392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319224.541445:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1f80 x1853163985115520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319224.541518:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319224.541562:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985115648:0@lo:400:kworker.0 00000100:00100000:2.0:1767319224.541610:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985115136:0@lo:400:kworker.0 00000100:00100000:1.0:1767319224.541654:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319224.541666:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319224.541711:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319224.541721:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985115776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319224.541735:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115392 00000100:00100000:2.0:1767319224.541745:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115392 00000100:00100000:1.0:1767319224.541762:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319224.541814:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115136 00000100:00100000:2.0:1767319224.541819:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115136 00000100:00100000:1.0:1767319224.541825:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115648 00000100:00100000:1.0:1767319224.541834:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115648 00000100:00100000:2.0:1767319224.541855:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4491 00000100:00100000:2.0:1767319224.541869:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985115392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319224.541882:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115776 00000100:00100000:1.0:1767319224.541888:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115776 00000100:00100000:1.0:1767319224.541925:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2959 00000100:00100000:1.0:1767319224.541936:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985115648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319224.542005:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985115648:12345-0@lo:400:kworker.0 Request processed in 70us (353us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319224.542013:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2959 00000100:00100000:0.0:1767319224.542021:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985114880:0@lo:38:kworker.0 00000100:00100000:1.0:1767319224.542034:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2960 00000100:00100000:1.0:1767319224.542043:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985115776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.542068:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319224.542095:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985115776:12345-0@lo:400:kworker.0 Request processed in 51us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319224.542102:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2960 00000100:00100000:0.0:1767319224.542109:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985115008:0@lo:38:kworker.0 00000100:00100000:1.0:1767319224.542131:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985114880 00000100:00100000:1.0:1767319224.542138:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985114880 00000100:00100000:0.0:1767319224.542155:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319224.542220:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4492 00000100:00100000:1.0:1767319224.542231:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985115136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.542248:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985115264:0@lo:38:kworker.0 00000100:00080000:3.0:1767319224.542346:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319224.542393:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115008 00000100:00100000:3.0:1767319224.542398:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115008 00000100:00100000:3.0:1767319224.542423:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1721 00000100:00100000:3.0:1767319224.542429:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985115008:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319224.542452:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319224.543820:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319224.543848:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985115520:0@lo:38:kworker.0 00000100:00100000:0.0:1767319224.543881:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319224.543963:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985115776:0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.543976:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985115648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.544009:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115264 00000100:00100000:0.0:1767319224.544015:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115264 00000100:00100000:0.0:1767319224.544038:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115520 00000100:00100000:0.0:1767319224.544040:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115520 00000100:00100000:0.0:1767319224.544052:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4924 00000100:00100000:0.0:1767319224.544058:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985114880:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319224.544077:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319224.544117:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985114880:12345-0@lo:38:kworker.0 Request processed in 58us (2051us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319224.544125:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4924 00000100:00100000:0.0:1767319224.544139:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4925 00000100:00100000:0.0:1767319224.544143:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985115264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319224.544152:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319224.544193:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985115264:12345-0@lo:38:kworker.0 Request processed in 49us (379us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319224.544199:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4925 00000100:00100000:0.0:1767319224.544213:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4926 00000100:00100000:0.0:1767319224.544217:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985115520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319224.544225:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319224.544246:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985115520:12345-0@lo:38:kworker.0 Request processed in 28us (364us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319224.544252:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4926 00000100:00080000:0.0:1767319224.544275:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319224.544281:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319224.544284:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985115264:0@lo:38:kworker.0 00000100:00080000:0.0:1767319224.544292:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319224.544295:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319224.544298:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985115520:0@lo:38:kworker.0 00000100:00080000:0.0:1767319224.544307:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319224.544312:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319224.544314:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985114880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319224.544363:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985115392:0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.544375:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985115136:0@lo:400:kworker.0 00000100:00100000:1.0:1767319224.551449:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985115136:12345-0@lo:400:kworker.0 Request processed in 9204us (9768us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319224.551470:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4492 00000100:00100000:3.0:1767319224.561582:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985115008:12345-0@lo:38:kworker.0 Request processed in 19137us (19414us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319224.561601:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1721 00000100:00080000:0.0:1767319224.561678:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319224.561692:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319224.561696:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985115008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319224.561796:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985115392:12345-0@lo:400:kworker.0 Request processed in 19927us (20279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319224.561808:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4491 00010000:00010000:0.0:1767319225.179656:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319226.205148:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319227.228340:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319228.123430:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985115904:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319228.123530:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319228.123596:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985115904 00000100:00100000:2.0:1767319228.123610:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319228.123611:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985115904 00000100:00100000:2.0:1767319228.123625:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985116032:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319228.123653:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319228.123701:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116032 00000100:00100000:1.0:1767319228.123707:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116032 00000100:00100000:1.0:1767319228.123748:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2122 00000100:00100000:1.0:1767319228.123760:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985115904:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319228.123813:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319228.123892:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985115904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 134us (365us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319228.123900:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2122 00000100:00100000:1.0:1767319228.123919:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2123 00000100:00100000:1.0:1767319228.123926:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985116032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319228.123944:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319228.123971:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985116032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 45us (317us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319228.123977:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2123 00000100:00080000:0.0:1767319228.124058:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319228.124061:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319228.124147:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985116032:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319228.124157:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985115904:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319228.253631:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319229.278571:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319229.617108:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319229.617161:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319229.620026:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548992 00000100:00100000:3.0:1767319229.620045:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548992 00000100:00100000:2.0:1767319229.620220:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920548864 00000100:00100000:2.0:1767319229.620225:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920548864 00000100:00100000:2.0:1767319229.620281:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4927 00000100:00100000:2.0:1767319229.620290:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920548864:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319229.620318:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319229.620403:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920548864:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 112us (3239us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319229.620411:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4927 00000100:00100000:3.0:1767319229.620458:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4493 00000100:00100000:3.0:1767319229.620469:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920548992:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319229.620555:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920548992:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 88us (3459us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319229.620563:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4493 00000100:00080000:0.0:1767319229.624046:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319229.660602:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319229.660615:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319229.660619:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319229.660623:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319229.660629:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319229.660639:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4472 00000100:00080000:1.0:1767319229.660644:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319229.660653:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319229.660706:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404380 x1853163985116160/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319229.660748:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319229.660752:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319229.660754:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319229.660758:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4472 00000100:00080000:1.0:1767319229.660761:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319229.660765:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319229.660783:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3800 x1853163985116288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319229.660870:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319229.661021:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985116160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319229.661106:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319229.661199:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985116288:0@lo:38:kworker.0 00000100:00100000:2.0:1767319229.661224:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319229.661271:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116160 00000100:00100000:2.0:1767319229.661280:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116160 00000100:00080000:1.0:1767319229.661587:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319229.661592:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319229.661594:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319229.661599:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4472 00000100:00080000:1.0:1767319229.661602:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319229.661605:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319229.661622:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0a80 x1853163985116544/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319229.661694:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4928 00000100:00100000:2.0:1767319229.661702:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985116160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319229.661726:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319229.661769:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985116160:12345-0@lo:38:kworker.0 Request processed in 67us (662us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319229.661777:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4928 00000100:00100000:2.0:1767319229.661805:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985116544:0@lo:38:kworker.0 00000100:00100000:2.0:1767319229.661839:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319229.661872:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319229.661877:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319229.661880:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985116160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319229.661908:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116544 00000100:00100000:2.0:1767319229.661912:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116544 00000100:00100000:2.0:1767319229.661932:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4929 00000100:00100000:2.0:1767319229.661937:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985116544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319229.661946:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319229.661972:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985116544:12345-0@lo:38:kworker.0 Request processed in 34us (132us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319229.661978:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4929 00000100:00080000:2.0:1767319229.661996:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319229.662000:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319229.662003:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985116544:0@lo:38:kworker.0 00000100:00100000:2.0:1767319229.662038:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116288 00000100:00100000:2.0:1767319229.662041:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116288 00000100:00100000:2.0:1767319229.662058:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1722 00000100:00100000:2.0:1767319229.662063:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985116288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319229.662073:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319229.662099:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985116288:12345-0@lo:38:kworker.0 Request processed in 35us (874us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319229.662106:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1722 00000100:00080000:2.0:1767319229.662124:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319229.662128:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319229.662131:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985116288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319229.663478:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319229.663505:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985116672:0@lo:400:kworker.0 00000100:00100000:0.0:1767319229.663574:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319229.664467:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116672 00000100:00100000:2.0:1767319229.664476:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116672 00000100:00100000:2.0:1767319229.664933:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4494 00000100:00100000:2.0:1767319229.664943:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985116672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319229.665021:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985116672:12345-0@lo:400:kworker.0 Request processed in 80us (1446us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319229.665029:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4494 00000100:00100000:2.0:1767319229.665076:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985116672:0@lo:400:kworker.0 00000100:00080000:0.0:1767319229.666926:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319229.668806:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985116416:0@lo:400:kworker.0 00000100:00100000:2.0:1767319229.668900:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319229.669120:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319229.669127:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319229.669131:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319229.669137:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4472 00000100:00080000:1.0:1767319229.669142:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319229.669148:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319229.669206:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2680 x1853163985116800/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319229.669214:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116416 00000100:00100000:3.0:1767319229.669226:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116416 00000100:00100000:1.0:1767319229.669286:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985116928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319229.669307:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4495 00000100:00100000:3.0:1767319229.669318:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985116416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319229.669385:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985116416:12345-0@lo:400:kworker.0 Request processed in 68us (484us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319229.669395:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4495 00000100:00100000:0.0:1767319229.669404:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985117056:0@lo:400:kworker.0 00000100:00080000:3.0:1767319229.669421:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319229.669449:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319229.669475:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117056 00000100:00100000:3.0:1767319229.669481:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117056 00000100:00100000:0.0:1767319229.669513:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985116800:0@lo:38:kworker.0 00000100:00100000:3.0:1767319229.669514:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2961 00000100:00100000:3.0:1767319229.669535:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985117056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319229.669554:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319229.669586:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985117056:12345-0@lo:400:kworker.0 Request processed in 62us (137us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319229.669594:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2961 00000100:00100000:0.0:1767319229.669612:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985117056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319229.669622:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116800 00000100:00100000:3.0:1767319229.669627:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116800 00000100:00100000:3.0:1767319229.669648:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4930 00000100:00100000:3.0:1767319229.669654:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985116800:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319229.669675:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319229.669717:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985116800:12345-0@lo:38:kworker.0 Request processed in 62us (164us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319229.669734:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4930 00000100:00080000:0.0:1767319229.669896:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319229.669901:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319229.669905:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985116800:0@lo:38:kworker.0 00000100:00100000:1.0:1767319229.669965:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319229.670351:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985116928 00000100:00100000:3.0:1767319229.670356:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985116928 00000100:00100000:3.0:1767319229.670388:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2962 00000100:00100000:3.0:1767319229.670396:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985116928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319229.670439:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985116928:12345-0@lo:400:kworker.0 Request processed in 45us (473us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319229.670446:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2962 00000100:00100000:1.0:1767319229.670702:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985116928:0@lo:400:kworker.0 00000100:00100000:2.0:1767319229.670956:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985116416:0@lo:400:kworker.0 00010000:00010000:2.0:1767319230.300568:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319231.325954:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319232.347864:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319233.243883:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985117184:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319233.244229:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319233.244315:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319233.244322:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985117312:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319233.244650:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117184 00000100:00100000:2.0:1767319233.244659:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117184 00000100:00100000:2.0:1767319233.244737:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2124 00000100:00100000:2.0:1767319233.244748:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985117184:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319233.244779:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319233.244827:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985117184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (599us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319233.244836:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2124 00000100:00080000:1.0:1767319233.244905:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319233.245162:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319233.245240:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985117184:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319233.245328:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117312 00000100:00100000:0.0:1767319233.245334:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117312 00000100:00080000:1.0:1767319233.245561:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319233.245893:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2125 00000100:00100000:0.0:1767319233.245903:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985117312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319233.245924:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319233.246130:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985117312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 229us (967us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319233.246138:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2125 00000100:00100000:3.0:1767319233.246364:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985117312:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319233.371696:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319234.399225:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319234.736669:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319234.736853:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319234.737009:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920549248 00000100:00100000:3.0:1767319234.737020:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920549248 00000100:00100000:3.0:1767319234.737249:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920549120 00000100:00100000:3.0:1767319234.737254:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920549120 00000100:00100000:3.0:1767319234.737289:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4496 00000100:00100000:3.0:1767319234.737302:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920549248:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319234.737392:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920549248:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (733us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319234.737408:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4496 00000100:00100000:3.0:1767319234.737436:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4931 00000100:00100000:3.0:1767319234.737441:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920549120:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319234.737466:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319234.737495:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920549120:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (642us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319234.737502:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4931 00000100:00080000:0.0:1767319234.740712:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319234.779435:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319234.779447:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319234.779450:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319234.779454:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319234.779458:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319234.779466:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4477 00000100:00080000:1.0:1767319234.779471:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319234.779478:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319234.779525:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0000 x1853163985117440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319234.779552:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319234.779555:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319234.779557:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319234.779561:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4477 00000100:00080000:1.0:1767319234.779564:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319234.779567:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319234.779585:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2a00 x1853163985117568/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319234.779623:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319234.779626:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319234.779628:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319234.779631:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4477 00000100:00080000:1.0:1767319234.779634:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319234.779637:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319234.779648:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2300 x1853163985117824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319234.779664:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319234.779667:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319234.779669:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319234.779672:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4477 00000100:00080000:1.0:1767319234.779674:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319234.779677:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319234.779692:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2680 x1853163985118080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319234.779767:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985118208:0@lo:400:kworker.0 00000100:00100000:1.0:1767319234.779829:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319234.779891:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319234.779898:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985118336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319234.779920:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319234.779954:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118208 00000100:00100000:1.0:1767319234.779961:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118208 00000100:00100000:1.0:1767319234.780000:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118336 00000100:00100000:1.0:1767319234.780003:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118336 00000100:00100000:1.0:1767319234.780022:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2963 00000100:00100000:1.0:1767319234.780030:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985118208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319234.780083:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985118208:12345-0@lo:400:kworker.0 Request processed in 55us (255us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319234.780089:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2963 00000100:00100000:1.0:1767319234.780101:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2964 00000100:00100000:1.0:1767319234.780104:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985118336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319234.780119:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985118336:12345-0@lo:400:kworker.0 Request processed in 15us (200us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319234.780122:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2964 00000100:00100000:1.0:1767319234.780152:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985118336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319234.780158:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985118208:0@lo:400:kworker.0 00000100:00100000:3.0:1767319234.780755:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985117952:0@lo:400:kworker.0 00000100:00100000:3.0:1767319234.780796:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319234.780837:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319234.780843:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985117696:0@lo:400:kworker.0 00000100:00100000:3.0:1767319234.780863:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319234.781201:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985117440:0@lo:38:kworker.0 00000100:00100000:0.0:1767319234.781233:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319234.781529:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117952 00000100:00100000:2.0:1767319234.781536:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117952 00000100:00100000:2.0:1767319234.781585:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117696 00000100:00100000:2.0:1767319234.781588:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117696 00000100:00100000:0.0:1767319234.781593:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985117568:0@lo:38:kworker.0 00000100:00100000:2.0:1767319234.781609:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117440 00000100:00100000:2.0:1767319234.781612:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117440 00000100:00100000:0.0:1767319234.781621:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319234.781627:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4497 00000100:00100000:2.0:1767319234.781634:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6ada300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985117952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319234.781643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985117824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319234.781663:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319234.781673:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985118080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319234.781679:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6ada300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985117952:12345-0@lo:400:kworker.0 Request processed in 46us (883us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319234.781686:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4497 00000100:00100000:0.0:1767319234.781691:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319234.781699:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117824 00000100:00100000:2.0:1767319234.781702:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117824 00000100:00100000:2.0:1767319234.781712:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118080 00000100:00100000:2.0:1767319234.781714:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118080 00000100:00100000:0.0:1767319234.781715:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4498 00000100:00100000:0.0:1767319234.781791:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6adaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985117696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319234.781822:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6adaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985117696:12345-0@lo:400:kworker.0 Request processed in 35us (958us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319234.786729:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4932 00000100:00100000:2.0:1767319234.786743:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985117440:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319234.786779:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319234.787192:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985117440:12345-0@lo:38:kworker.0 Request processed in 447us (5955us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319234.787200:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4932 00000100:00100000:2.0:1767319234.787219:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4933 00000100:00100000:2.0:1767319234.787223:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985117824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319234.787234:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319234.787258:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985117824:12345-0@lo:38:kworker.0 Request processed in 33us (5594us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319234.787263:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4933 00000100:00100000:2.0:1767319234.787274:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4934 00000100:00100000:2.0:1767319234.787278:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985118080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319234.787287:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319234.787494:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985118080:12345-0@lo:38:kworker.0 Request processed in 115us (5702us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319234.787501:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4934 00000100:00080000:2.0:1767319234.787536:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319234.787609:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985117696:0@lo:400:kworker.0 00000100:00100000:2.0:1767319234.787626:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985117952:0@lo:400:kworker.0 00000100:00100000:2.0:1767319234.787670:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985117568 00000100:00100000:2.0:1767319234.787678:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985117568 00000100:00100000:2.0:1767319234.787707:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1723 00000100:00100000:2.0:1767319234.787712:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985117568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319234.787721:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319234.787936:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985117568:12345-0@lo:38:kworker.0 Request processed in 222us (6313us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319234.787943:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1723 00000100:00080000:2.0:1767319234.787991:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319234.788088:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319234.788092:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985117440:0@lo:38:kworker.0 00000100:00080000:2.0:1767319234.788106:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319234.788108:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319234.788112:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985117568:0@lo:38:kworker.0 00000100:00080000:2.0:1767319234.788119:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319234.788122:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319234.788125:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985117824:0@lo:38:kworker.0 00000100:00080000:2.0:1767319234.788133:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319234.788136:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319234.788138:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985118080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319234.798115:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4498 00010000:00010000:2.0:1767319235.419327:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319236.443688:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319237.468556:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319238.363728:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985118592:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319238.363827:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319238.363913:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319238.363920:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985118464:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319238.363944:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319238.364022:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118592 00000100:00100000:1.0:1767319238.364031:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118592 00000100:00100000:1.0:1767319238.364093:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118464 00000100:00100000:1.0:1767319238.364097:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118464 00000100:00100000:1.0:1767319238.364122:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2126 00000100:00100000:1.0:1767319238.364131:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985118592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319238.364159:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319238.364228:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985118592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (401us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319238.364236:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2126 00000100:00100000:1.0:1767319238.364254:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2127 00000100:00100000:1.0:1767319238.364261:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985118464:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319238.364274:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319238.364300:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985118464:12345-0@lo:13:osp-pre-1-1.0 Request processed in 40us (355us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319238.364306:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2127 00000100:00080000:0.0:1767319238.364572:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319238.364577:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319238.364610:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985118464:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319238.364622:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985118592:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319238.491922:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319239.515454:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319239.858043:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319239.858085:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920549376 offset 0 length 520: 4 00000100:00100000:2.0:1767319239.858113:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319239.858882:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920549632 00000100:00100000:0.0:1767319239.858894:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920549632 00000100:00100000:0.0:1767319239.858967:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920549504 00000100:00100000:0.0:1767319239.858971:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920549504 00000100:00100000:3.0:1767319239.858976:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4499 00000100:00100000:3.0:1767319239.858987:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920549632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319239.858990:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4935 00000100:00100000:0.0:1767319239.858995:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920549504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319239.859014:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319239.859061:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920549632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (1030us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319239.859069:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4499 00000100:00080000:2.0:1767319239.859395:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319239.871506:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920549504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 12496us (13377us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319239.871521:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4935 00000100:00080000:3.0:1767319239.899512:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319239.899522:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319239.899525:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319239.899529:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319214/real 1767319214] req@ffff8ce1a612ce00 x1853163985112192/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319239 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319239.899529:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319239.899533:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319239.899539:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4483 00000100:00080000:3.0:1767319239.899542:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319239.899547:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319239.899560:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a612ce00 x1853163985112192/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319239 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319239.899571:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319239.899586:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1c00 x1853163985118720/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319239.899595:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319239.899598:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985112192:0@lo:250:kworker.0 00000100:00080000:3.0:1767319239.899600:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319239.899603:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319239.899605:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319239.899607:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4483 00000100:00080000:3.0:1767319239.899609:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319239.899612:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319239.899621:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985118720:0@lo:38:kworker.0 00000100:00100000:3.0:1767319239.899623:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1f80 x1853163985118848/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319239.899661:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319239.899665:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319239.899666:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319239.899669:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4483 00000100:00080000:3.0:1767319239.899672:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319239.899674:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319239.899677:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319239.899684:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1180 x1853163985119104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319239.899699:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319239.899701:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319239.899703:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319239.899706:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4483 00000100:00080000:3.0:1767319239.899707:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319239.899710:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319239.899719:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985119360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319239.899732:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985118848:0@lo:38:kworker.0 00000100:00100000:2.0:1767319239.899762:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319239.899776:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319239.899779:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319239.899780:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319239.899783:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4462 00000100:00100000:2.0:1767319239.899784:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985119104:0@lo:38:kworker.0 00000100:00080000:3.0:1767319239.899786:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319239.899789:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319239.899798:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985119744/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319239.899803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319239.899818:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985119360:0@lo:38:kworker.0 00000100:00100000:3.0:1767319239.899818:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985119488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319239.899837:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319239.899844:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319239.899855:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985119744:0@lo:250:kworker.0 00000100:00100000:3.0:1767319239.899860:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319239.899864:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985119616:0@lo:400:kworker.0 00000400:00000100:2.0:1767319239.899871:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985119744 offset 0 length 520: 4 00000100:00100000:3.0:1767319239.899881:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319239.899903:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118720 00000100:00100000:3.0:1767319239.899909:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118720 00000100:00100000:2.0:1767319239.899911:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118848 00000100:00100000:2.0:1767319239.899914:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118848 00000100:00100000:3.0:1767319239.899935:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985119104 00000100:00100000:3.0:1767319239.899937:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985119104 00000100:00100000:2.0:1767319239.899939:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1724 00000100:00100000:2.0:1767319239.899946:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985118848:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319239.899948:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985119360 00000100:00100000:3.0:1767319239.899950:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985119360 00000100:00100000:3.0:1767319239.899963:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4936 00010000:00020000:2.0:1767319239.899967:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319239.899967:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985118720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319239.899976:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319239.900007:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985118848:12345-0@lo:38:kworker.0 Request processed in 60us (243us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319239.900016:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1724 00000100:00100000:2.0:1767319239.900044:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4937 00000100:00100000:2.0:1767319239.900048:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985119104:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319239.900057:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319239.900102:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985118720:12345-0@lo:38:kworker.0 Request processed in 133us (424us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319239.900108:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4936 00000100:00100000:3.0:1767319239.900121:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4938 00000100:00100000:3.0:1767319239.900124:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985119360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319239.900133:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319239.900156:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985119360:12345-0@lo:38:kworker.0 Request processed in 30us (318us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319239.900162:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4938 00000100:00080000:3.0:1767319239.900219:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319239.900224:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319239.900227:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985118848:0@lo:38:kworker.0 00000100:00100000:1.0:1767319239.900276:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985119232:0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900303:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319239.900353:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319239.900358:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985118976:0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900383:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319239.900407:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985119488 00000100:00100000:1.0:1767319239.900411:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985119488 00000100:00100000:1.0:1767319239.900440:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985119616 00000100:00100000:1.0:1767319239.900442:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985119616 00000100:00100000:1.0:1767319239.900459:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2965 00000100:00100000:1.0:1767319239.900465:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985119488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900505:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985119488:12345-0@lo:400:kworker.0 Request processed in 40us (659us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319239.900511:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2965 00000100:00100000:1.0:1767319239.900523:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2966 00000100:00100000:1.0:1767319239.900529:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985119616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900555:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985119616:12345-0@lo:400:kworker.0 Request processed in 27us (673us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319239.900560:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2966 00000100:00100000:1.0:1767319239.900592:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985119232 00000100:00100000:1.0:1767319239.900595:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985119232 00000100:00100000:1.0:1767319239.900621:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985118976 00000100:00100000:1.0:1767319239.900624:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985118976 00000100:00100000:1.0:1767319239.900639:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4500 00000100:00100000:1.0:1767319239.900645:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985119232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900849:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985119232:12345-0@lo:400:kworker.0 Request processed in 204us (544us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319239.900855:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4500 00000100:00100000:1.0:1767319239.900868:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4501 00000100:00100000:1.0:1767319239.900873:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985118976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900899:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985118976:12345-0@lo:400:kworker.0 Request processed in 27us (516us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319239.900905:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4501 00000100:00100000:1.0:1767319239.900935:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985119616:0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900945:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985119488:0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900969:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985118976:0@lo:400:kworker.0 00000100:00100000:1.0:1767319239.900977:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985119232:0@lo:400:kworker.0 00000100:00080000:1.0:1767319239.901007:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319239.906053:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985119104:12345-0@lo:38:kworker.0 Request processed in 5994us (6238us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767319239.906053:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319239.906065:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319239.906070:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4937 00000100:00100000:3.0:1767319239.906074:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985119104:0@lo:38:kworker.0 00000100:00080000:3.0:1767319239.906257:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319239.906261:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319239.906265:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985119360:0@lo:38:kworker.0 00000100:00080000:3.0:1767319239.906278:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319239.906282:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319239.906285:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985118720:0@lo:38:kworker.0 00010000:00010000:1.0:1767319240.539480:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319241.563984:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319242.587741:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319243.483533:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319243.483567:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985119872:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319243.483675:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319243.483741:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985120000:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319243.483757:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985119872 00000100:00100000:0.0:1767319243.483768:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319243.483771:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985119872 00000100:00080000:0.0:1767319243.483826:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319243.483868:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120000 00000100:00100000:2.0:1767319243.483873:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120000 00000100:00100000:2.0:1767319243.483906:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2128 00000100:00100000:2.0:1767319243.483916:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985119872:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319243.483948:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319243.484042:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985119872:12345-0@lo:13:osp-pre-0-1.0 Request processed in 125us (364us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319243.484050:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2128 00000100:00080000:3.0:1767319243.484058:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319243.484090:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2129 00000100:00100000:3.0:1767319243.484107:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985120000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319243.484137:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319243.484233:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985119872:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319243.484261:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985120000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 159us (488us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319243.484375:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985120000:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319243.485841:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2129 00010000:00010000:0.0:1767319243.615899:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319244.636669:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319244.977210:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319244.977278:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319244.977503:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920549888 00000100:00100000:2.0:1767319244.977518:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920549888 00000100:00100000:2.0:1767319244.977633:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920549760 00000100:00100000:2.0:1767319244.977638:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920549760 00000100:00100000:2.0:1767319244.977659:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4502 00000100:00100000:2.0:1767319244.977672:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920549888:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319244.977788:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920549888:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 119us (628us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319244.977801:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4502 00000100:00100000:2.0:1767319244.977823:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4939 00000100:00100000:2.0:1767319244.977828:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920549760:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319244.977855:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319244.977883:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920549760:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (605us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319244.977890:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4939 00000100:00080000:1.0:1767319244.979756:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319245.021092:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319245.021106:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319245.021109:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319245.021114:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319245.021119:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319245.021138:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4488 00000100:00080000:0.0:1767319245.021145:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319245.021155:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319245.021251:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4000 x1853163985120128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319245.021281:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319245.021284:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319245.021286:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319245.021290:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4488 00000100:00080000:0.0:1767319245.021292:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319245.021297:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319245.021310:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5880 x1853163985120256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319245.021359:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319245.021364:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319245.021366:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319245.021370:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4488 00000100:00080000:0.0:1767319245.021373:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319245.021376:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319245.021390:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6680 x1853163985120512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319245.021406:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319245.021410:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319245.021412:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319245.021415:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4488 00000100:00080000:0.0:1767319245.021418:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319245.021421:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319245.021433:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985120768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319245.021486:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985121024:0@lo:400:kworker.0 00000100:00100000:1.0:1767319245.021854:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985120384:0@lo:400:kworker.0 00000100:00100000:1.0:1767319245.021925:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319245.022005:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319245.022012:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985120640:0@lo:400:kworker.0 00000100:00100000:1.0:1767319245.022035:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319245.022070:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120384 00000100:00100000:1.0:1767319245.022079:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120384 00000100:00100000:1.0:1767319245.022157:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120640 00000100:00100000:1.0:1767319245.022161:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120640 00000100:00100000:1.0:1767319245.022225:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4503 00000100:00100000:1.0:1767319245.022234:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985120384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319245.022310:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985120384:12345-0@lo:400:kworker.0 Request processed in 76us (385us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319245.022318:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4503 00000100:00100000:1.0:1767319245.022339:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4504 00000100:00100000:1.0:1767319245.022345:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985120640:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319245.022372:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985120640:12345-0@lo:400:kworker.0 Request processed in 28us (337us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319245.022377:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4504 00000100:00100000:1.0:1767319245.022404:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985120896:0@lo:400:kworker.0 00000100:00100000:1.0:1767319245.022435:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319245.022470:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120896 00000100:00100000:1.0:1767319245.022474:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120896 00000100:00100000:1.0:1767319245.022509:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2967 00000100:00100000:1.0:1767319245.022515:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985120896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319245.022909:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319245.022973:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319245.022979:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319245.023016:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985120640:0@lo:400:kworker.0 00000100:00100000:0.0:1767319245.023028:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985120384:0@lo:400:kworker.0 00000100:00100000:0.0:1767319245.023082:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121024 00000100:00100000:0.0:1767319245.023087:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121024 00000100:00100000:0.0:1767319245.023122:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2968 00000100:00100000:0.0:1767319245.023128:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985121024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319245.023212:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985121024:12345-0@lo:400:kworker.0 Request processed in 83us (304us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319245.023219:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2968 00000100:00080000:0.0:1767319245.023238:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319245.023266:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985121024:0@lo:400:kworker.0 00000100:00100000:3.0:1767319245.023850:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985120128:0@lo:38:kworker.0 00000100:00100000:3.0:1767319245.023904:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319245.023952:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985120256:0@lo:38:kworker.0 00000100:00100000:3.0:1767319245.023979:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319245.024018:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985120512:0@lo:38:kworker.0 00000100:00100000:3.0:1767319245.024042:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319245.024061:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985120768:0@lo:38:kworker.0 00000100:00100000:3.0:1767319245.024088:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319245.024129:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120256 00000100:00100000:3.0:1767319245.024135:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120256 00000100:00100000:3.0:1767319245.024201:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1725 00000100:00100000:3.0:1767319245.024208:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985120256:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319245.024233:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319245.024285:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985120256:12345-0@lo:38:kworker.0 Request processed in 76us (305us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319245.024293:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1725 00000100:00100000:3.0:1767319245.024321:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120128 00000100:00100000:3.0:1767319245.024325:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120128 00000100:00100000:3.0:1767319245.024344:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120512 00000100:00100000:3.0:1767319245.024347:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120512 00000100:00100000:3.0:1767319245.024359:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985120768 00000100:00100000:3.0:1767319245.024362:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985120768 00000100:00100000:3.0:1767319245.024378:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4940 00000100:00100000:3.0:1767319245.024383:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985120128:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319245.024392:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319245.024420:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985120128:12345-0@lo:38:kworker.0 Request processed in 36us (515us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319245.024426:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4940 00000100:00100000:3.0:1767319245.024436:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4941 00000100:00100000:3.0:1767319245.024440:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985120512:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319245.024450:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319245.024471:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985120512:12345-0@lo:38:kworker.0 Request processed in 30us (428us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319245.024481:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4941 00000100:00100000:3.0:1767319245.024491:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4942 00000100:00100000:3.0:1767319245.024495:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985120768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319245.024503:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319245.025161:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985120896:12345-0@lo:400:kworker.0 Request processed in 2641us (2721us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319245.025186:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2967 00000100:00100000:1.0:1767319245.025260:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985120896:0@lo:400:kworker.0 00000100:00080000:1.0:1767319245.025336:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319245.025343:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319245.025347:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985120128:0@lo:38:kworker.0 00000100:00080000:1.0:1767319245.025356:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319245.025359:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319245.025362:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985120256:0@lo:38:kworker.0 00000100:00080000:1.0:1767319245.025371:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319245.025374:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319245.025377:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985120512:0@lo:38:kworker.0 00000100:00080000:1.0:1767319245.025386:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319245.025389:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319245.025391:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985120768:0@lo:38:kworker.0 00000100:00100000:3.0:1767319245.025821:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985120768:12345-0@lo:38:kworker.0 Request processed in 29us (437us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319245.025834:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4942 00010000:00010000:1.0:1767319245.660745:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319246.685985:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319247.707751:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319248.603482:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985121152:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319248.603550:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319248.603719:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121152 00000100:00100000:1.0:1767319248.603742:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121152 00000100:00100000:1.0:1767319248.603801:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2130 00000100:00100000:1.0:1767319248.603811:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985121152:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319248.603856:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319248.603901:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985121152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 91us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319248.603908:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2130 00000100:00080000:0.0:1767319248.604062:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319248.604110:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985121152:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319248.604639:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319248.604650:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985121280:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319248.604683:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319248.604903:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121280 00000100:00100000:2.0:1767319248.604909:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121280 00000100:00100000:2.0:1767319248.604957:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2131 00000100:00100000:2.0:1767319248.604967:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985121280:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319248.604986:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319248.605031:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985121280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (347us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319248.605037:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2131 00000100:00100000:3.0:1767319248.605215:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985121280:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319248.605394:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319248.732199:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319249.755418:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319250.094928:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319250.094976:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319250.095038:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920550144 00000100:00100000:3.0:1767319250.095049:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920550144 00000100:00100000:0.0:1767319250.095075:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920550016 00000100:00100000:0.0:1767319250.095081:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920550016 00000100:00100000:0.0:1767319250.095141:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4505 00000100:00080000:1.0:1767319250.095156:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319250.095226:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920550144:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319250.095311:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920550144:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 162us (393us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319250.095327:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4505 00000100:00100000:0.0:1767319250.095347:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4943 00000100:00100000:0.0:1767319250.095353:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920550016:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319250.095372:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319250.095398:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920550016:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (421us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319250.095404:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4943 00000100:00080000:3.0:1767319250.139686:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319250.139698:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319250.139701:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319250.139705:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319250.139710:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319250.139718:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4493 00000100:00080000:3.0:1767319250.139722:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319250.139729:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319250.139766:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07b80 x1853163985121408/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319250.139826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985121408:0@lo:38:kworker.0 00000100:00100000:1.0:1767319250.139894:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319250.139980:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319250.139985:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319250.139987:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319250.139992:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4493 00000100:00080000:3.0:1767319250.139995:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319250.139999:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319250.140013:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05c00 x1853163985121536/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319250.140059:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121408 00000100:00100000:0.0:1767319250.140067:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121408 00000100:00100000:0.0:1767319250.140108:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4944 00000100:00100000:0.0:1767319250.140115:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985121408:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319250.140135:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319250.140369:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985121536:0@lo:38:kworker.0 00000100:00080000:3.0:1767319250.140393:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319250.140396:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767319250.140397:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319250.140398:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319250.140402:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4493 00000100:00080000:3.0:1767319250.140404:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319250.140408:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319250.140419:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05180 x1853163985121792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319250.140475:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985121664:0@lo:400:kworker.0 00000100:00100000:2.0:1767319250.140512:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319250.140545:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121664 00000100:00100000:2.0:1767319250.140549:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121664 00000100:00100000:2.0:1767319250.141922:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4506 00000100:00100000:2.0:1767319250.141941:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985121664:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319250.142023:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985121664:12345-0@lo:400:kworker.0 Request processed in 91us (1509us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319250.142033:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4506 00000100:00080000:2.0:1767319250.142064:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319250.142104:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985121664:0@lo:400:kworker.0 00000100:00100000:3.0:1767319250.142413:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121536 00000100:00100000:3.0:1767319250.142420:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121536 00000100:00100000:3.0:1767319250.142449:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1726 00000100:00100000:3.0:1767319250.142454:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985121536:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319250.142471:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319250.142517:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985121536:12345-0@lo:38:kworker.0 Request processed in 61us (2118us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319250.142523:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1726 00000100:00080000:2.0:1767319250.143077:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319250.143081:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319250.143084:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319250.143089:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4493 00000100:00080000:2.0:1767319250.143093:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319250.143097:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319250.143141:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5500 x1853163985122048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319250.143222:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319250.143233:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985121920:0@lo:400:kworker.0 00000100:00100000:2.0:1767319250.143271:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319250.143301:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121920 00000100:00100000:2.0:1767319250.143306:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121920 00000100:00100000:2.0:1767319250.143356:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4507 00000100:00100000:2.0:1767319250.143364:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985121920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319250.143402:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985121920:12345-0@lo:400:kworker.0 Request processed in 38us (130us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319250.143408:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4507 00000100:00080000:2.0:1767319250.143424:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319250.143443:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985121920:0@lo:400:kworker.0 00000100:00100000:0.0:1767319250.143761:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985121408:12345-0@lo:38:kworker.0 Request processed in 3643us (3864us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319250.143771:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4944 00000100:00100000:3.0:1767319250.143851:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985122304:0@lo:400:kworker.0 00000100:00100000:3.0:1767319250.143885:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319250.143915:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319250.143921:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985122176:0@lo:400:kworker.0 00000100:00100000:3.0:1767319250.143941:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319250.143986:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122304 00000100:00100000:2.0:1767319250.143991:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122304 00000100:00100000:0.0:1767319250.144011:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122176 00000100:00100000:0.0:1767319250.144015:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122176 00000100:00100000:2.0:1767319250.144039:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2969 00000100:00100000:0.0:1767319250.144045:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2970 00000100:00100000:2.0:1767319250.144050:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985122304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319250.144055:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985122176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319250.144098:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985122176:12345-0@lo:400:kworker.0 Request processed in 47us (156us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319250.144104:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985122304:12345-0@lo:400:kworker.0 Request processed in 55us (215us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319250.144105:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2970 00000100:00100000:2.0:1767319250.144116:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2969 00000100:00080000:3.0:1767319250.144310:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319250.144313:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319250.144335:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985122176:0@lo:400:kworker.0 00000100:00100000:3.0:1767319250.144342:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985122304:0@lo:400:kworker.0 00000100:00080000:1.0:1767319250.144766:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319250.144770:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319250.144773:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985121408:0@lo:38:kworker.0 00000100:00100000:1.0:1767319250.144786:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985121792:0@lo:38:kworker.0 00000100:00100000:1.0:1767319250.144809:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319250.144832:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985122048:0@lo:38:kworker.0 00000100:00100000:1.0:1767319250.144851:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319250.144867:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985121792 00000100:00100000:0.0:1767319250.144871:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985121792 00000100:00080000:1.0:1767319250.144876:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319250.144880:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319250.144882:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985121536:0@lo:38:kworker.0 00000100:00100000:0.0:1767319250.144899:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122048 00000100:00100000:0.0:1767319250.144902:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122048 00000100:00100000:0.0:1767319250.144919:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4945 00000100:00100000:0.0:1767319250.144925:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985121792:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319250.144931:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4946 00000100:00100000:1.0:1767319250.144936:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985122048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319250.144940:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767319250.144943:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319250.145078:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985121792:12345-0@lo:38:kworker.0 Request processed in 152us (266us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319250.145084:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4945 00000100:00080000:0.0:1767319250.146250:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319250.146258:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319250.146262:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985121792:0@lo:38:kworker.0 00000100:00100000:1.0:1767319250.146349:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985122048:12345-0@lo:38:kworker.0 Request processed in 1410us (1495us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319250.146356:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4946 00000100:00080000:0.0:1767319250.146419:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319250.146424:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319250.146426:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985122048:0@lo:38:kworker.0 00010000:00010000:1.0:1767319250.792534:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319251.803969:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319252.827475:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319253.723649:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319253.723744:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319253.723824:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985122432:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319253.723892:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319253.724480:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122432 00000100:00100000:1.0:1767319253.724489:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122432 00000100:00100000:1.0:1767319253.724563:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2132 00000100:00100000:1.0:1767319253.724574:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985122432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319253.724603:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319253.724654:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985122432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (761us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319253.724661:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2132 00000100:00080000:3.0:1767319253.724984:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319253.725120:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985122432:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319253.725720:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985122560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319253.725768:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319253.725895:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122560 00000100:00100000:2.0:1767319253.725902:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122560 00000100:00100000:2.0:1767319253.726199:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2133 00000100:00100000:2.0:1767319253.726209:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985122560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319253.726230:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319253.726279:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985122560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (510us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319253.726287:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2133 00000100:00100000:0.0:1767319253.726365:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985122560:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319253.726436:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319253.851415:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319254.875625:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319255.213741:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319255.213853:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319255.213966:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920550272 00000100:00100000:1.0:1767319255.213977:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920550272 00000100:00100000:1.0:1767319255.214023:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920550400 00000100:00100000:1.0:1767319255.214026:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920550400 00000100:00100000:1.0:1767319255.214087:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4508 00000100:00100000:1.0:1767319255.214099:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920550400:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319255.214198:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920550400:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 102us (343us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319255.214207:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4508 00000100:00100000:1.0:1767319255.214226:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4947 00000100:00100000:1.0:1767319255.214242:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920550272:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319255.214262:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319255.214290:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920550272:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (561us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319255.214297:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4947 00000100:00080000:2.0:1767319255.214562:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319255.259351:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319255.259361:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319255.259365:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319255.259369:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319255.259373:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319255.259381:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4498 00000100:00080000:0.0:1767319255.259386:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319255.259393:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319255.259432:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4380 x1853163985122688/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319255.259458:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319255.259462:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319255.259464:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319255.259467:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4498 00000100:00080000:0.0:1767319255.259470:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319255.259473:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319255.259504:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5f80 x1853163985122816/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319255.259549:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319255.259553:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319255.259555:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319255.259559:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4498 00000100:00080000:0.0:1767319255.259561:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319255.259564:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319255.259820:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319255.259839:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985122944:0@lo:400:kworker.0 00000100:00100000:2.0:1767319255.259907:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319255.260032:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122944 00000100:00100000:3.0:1767319255.260041:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122944 00000100:00100000:3.0:1767319255.260116:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4509 00000100:00100000:3.0:1767319255.260126:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985122944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.260205:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985122944:12345-0@lo:400:kworker.0 Request processed in 80us (297us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319255.260213:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4509 00000100:00100000:3.0:1767319255.260261:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985122944:0@lo:400:kworker.0 00000100:00080000:2.0:1767319255.260325:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319255.260408:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7800 x1853163985123072/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319255.260440:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319255.260443:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319255.260445:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319255.260449:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4498 00000100:00080000:0.0:1767319255.260451:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319255.260456:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319255.260467:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7b80 x1853163985123328/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319255.260605:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319255.260615:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985123200:0@lo:400:kworker.0 00000100:00100000:2.0:1767319255.260643:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319255.260894:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123200 00000100:00100000:3.0:1767319255.260899:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123200 00000100:00100000:3.0:1767319255.260939:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4510 00000100:00100000:3.0:1767319255.260946:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985123200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.260983:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985123200:12345-0@lo:400:kworker.0 Request processed in 37us (339us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319255.260989:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4510 00000100:00100000:3.0:1767319255.261006:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985123584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.261031:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319255.261049:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319255.261069:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985123456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.261091:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319255.261117:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123584 00000100:00100000:3.0:1767319255.261120:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123584 00000100:00100000:3.0:1767319255.261146:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123456 00000100:00100000:3.0:1767319255.261149:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123456 00000100:00100000:3.0:1767319255.261186:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2971 00000100:00100000:3.0:1767319255.261193:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985123584:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.261227:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985123584:12345-0@lo:400:kworker.0 Request processed in 35us (195us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319255.261233:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2971 00000100:00100000:3.0:1767319255.261244:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2972 00000100:00100000:3.0:1767319255.261249:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985123456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.261273:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985123456:12345-0@lo:400:kworker.0 Request processed in 24us (181us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319255.261279:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2972 00000100:00100000:3.0:1767319255.261309:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985123456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.261318:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985123584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319255.261348:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985123200:0@lo:400:kworker.0 00000100:00100000:1.0:1767319255.261438:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985122688:0@lo:38:kworker.0 00000100:00100000:1.0:1767319255.261469:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319255.261511:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985122816:0@lo:38:kworker.0 00000100:00100000:1.0:1767319255.261535:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319255.261557:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985123072:0@lo:38:kworker.0 00000100:00100000:3.0:1767319255.261762:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122688 00000100:00100000:3.0:1767319255.261766:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122688 00000100:00100000:3.0:1767319255.261787:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4948 00000100:00100000:3.0:1767319255.261792:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985122688:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319255.261809:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319255.261836:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985122688:12345-0@lo:38:kworker.0 Request processed in 43us (367us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319255.261843:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4948 00000100:00100000:3.0:1767319255.261872:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985122816 00000100:00100000:3.0:1767319255.261875:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985122816 00000100:00100000:3.0:1767319255.261891:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1727 00000100:00100000:3.0:1767319255.261896:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985122816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319255.261905:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319255.261929:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985122816:12345-0@lo:38:kworker.0 Request processed in 32us (393us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319255.261935:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1727 00000100:00100000:1.0:1767319255.262014:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319255.262040:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985123328:0@lo:38:kworker.0 00000100:00100000:1.0:1767319255.262076:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319255.262103:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319255.262108:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319255.262111:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985122688:0@lo:38:kworker.0 00000100:00080000:1.0:1767319255.262118:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319255.262122:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319255.262125:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985122816:0@lo:38:kworker.0 00000100:00100000:1.0:1767319255.262146:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123072 00000100:00100000:1.0:1767319255.262149:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123072 00000100:00100000:1.0:1767319255.262183:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123328 00000100:00100000:1.0:1767319255.262185:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123328 00000100:00100000:1.0:1767319255.262200:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4949 00000100:00100000:1.0:1767319255.262205:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985123072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319255.262217:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319255.262247:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985123072:12345-0@lo:38:kworker.0 Request processed in 41us (232us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319255.262252:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4949 00000100:00100000:1.0:1767319255.262265:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4950 00000100:00100000:1.0:1767319255.262268:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985123328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319255.262277:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319255.262300:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985123328:12345-0@lo:38:kworker.0 Request processed in 29us (223us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319255.262306:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4950 00000100:00080000:1.0:1767319255.262330:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319255.262334:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319255.262337:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985123072:0@lo:38:kworker.0 00000100:00080000:1.0:1767319255.262344:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319255.262348:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319255.262350:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985123328:0@lo:38:kworker.0 00000100:00080000:0.0:1767319255.262433:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319255.262438:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319255.262440:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319255.900425:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319256.923729:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319257.947876:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319258.843522:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1767319258.843541:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985123840:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319258.843633:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319258.843700:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985123712:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319258.843722:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319258.843786:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123840 00000100:00100000:2.0:1767319258.843794:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123712 00000100:00100000:1.0:1767319258.843795:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123840 00000100:00100000:2.0:1767319258.843802:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123712 00000100:00100000:1.0:1767319258.843867:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2134 00000100:00100000:1.0:1767319258.843876:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985123840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319258.843906:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319258.843961:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985123840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (327us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319258.844014:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2134 00000100:00100000:3.0:1767319258.844129:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985123840:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319258.846392:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2135 00000100:00100000:2.0:1767319258.846410:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985123712:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319258.846442:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319258.846569:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985123712:12345-0@lo:13:osp-pre-1-1.0 Request processed in 165us (2844us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319258.846578:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2135 00000100:00100000:3.0:1767319258.846841:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985123712:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319258.846864:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319258.846871:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319258.971392:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319259.995517:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319260.341440:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319260.341496:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319260.341749:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920550528 00000100:00100000:0.0:1767319260.341759:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920550528 00000100:00100000:0.0:1767319260.341800:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920550656 00000100:00100000:0.0:1767319260.341804:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920550656 00000100:00100000:0.0:1767319260.341850:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4511 00000100:00100000:0.0:1767319260.341861:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920550656:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319260.341933:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920550656:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (436us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319260.341941:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4511 00000100:00100000:0.0:1767319260.341990:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4951 00000100:00100000:0.0:1767319260.341995:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920550528:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319260.342015:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319260.342049:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920550528:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (618us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319260.342054:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4951 00000100:00080000:3.0:1767319260.342619:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319260.379983:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319260.379992:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319260.379995:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319260.379999:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319260.380004:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319260.380012:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4503 00000100:00080000:2.0:1767319260.380017:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319260.380023:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319260.380070:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6680 x1853163985123968/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319260.380143:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985123968:0@lo:38:kworker.0 00000100:00100000:1.0:1767319260.380238:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319260.380375:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985123968 00000100:00100000:0.0:1767319260.380384:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985123968 00000100:00100000:0.0:1767319260.380434:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4952 00000100:00100000:0.0:1767319260.380441:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985123968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319260.380471:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319260.380555:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319260.380559:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319260.380561:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319260.380565:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4503 00000100:00080000:2.0:1767319260.380569:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319260.380573:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319260.380589:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4380 x1853163985124096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319260.383883:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985124096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319260.383977:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319260.384181:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985123968:12345-0@lo:38:kworker.0 Request processed in 3735us (3938us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319260.384195:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4952 00000100:00100000:3.0:1767319260.388681:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124096 00000100:00100000:3.0:1767319260.388708:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124096 00000100:00100000:3.0:1767319260.388769:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1728 00000100:00100000:3.0:1767319260.388777:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985124096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319260.388801:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319260.388885:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985124096:12345-0@lo:38:kworker.0 Request processed in 107us (4907us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319260.388893:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1728 00000100:00080000:1.0:1767319260.389258:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319260.389265:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319260.389269:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985123968:0@lo:38:kworker.0 00000100:00080000:1.0:1767319260.389300:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319260.389304:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319260.389307:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985124096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319260.389487:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1767319260.389609:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319260.389614:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319260.389617:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319260.389622:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4503 00000100:00080000:2.0:1767319260.389626:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319260.389632:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319260.389654:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6680 x1853163985124352/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319260.389691:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319260.389704:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319260.389706:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319260.389710:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4503 00000100:00080000:2.0:1767319260.389713:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319260.389716:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319260.389727:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7800 x1853163985124608/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319260.389781:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985124352:0@lo:38:kworker.0 00000100:00100000:1.0:1767319260.389834:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319260.389859:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985124608:0@lo:38:kworker.0 00000100:00100000:1.0:1767319260.389879:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319260.389907:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124352 00000100:00100000:1.0:1767319260.389911:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124352 00000100:00100000:1.0:1767319260.389932:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124608 00000100:00100000:1.0:1767319260.389934:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124608 00000100:00100000:1.0:1767319260.389950:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4953 00000100:00100000:1.0:1767319260.389955:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985124352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319260.389978:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319260.390010:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985124352:12345-0@lo:38:kworker.0 Request processed in 53us (175us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319260.390017:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4953 00000100:00100000:1.0:1767319260.390031:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4954 00000100:00100000:1.0:1767319260.390035:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985124608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319260.390043:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319260.390063:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985124608:12345-0@lo:38:kworker.0 Request processed in 27us (183us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319260.390068:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4954 00000100:00100000:3.0:1767319260.390293:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319260.390303:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985124864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390334:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.390363:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985124736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390384:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.390403:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319260.390408:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985124480:0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390451:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.390491:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124864 00000100:00100000:3.0:1767319260.390495:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124864 00000100:00100000:3.0:1767319260.390535:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124736 00000100:00100000:3.0:1767319260.390538:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124736 00000100:00100000:3.0:1767319260.390556:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2973 00000100:00100000:3.0:1767319260.390562:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985124864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390602:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985124864:12345-0@lo:400:kworker.0 Request processed in 40us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319260.390607:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2973 00000100:00100000:3.0:1767319260.390619:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2974 00000100:00100000:3.0:1767319260.390625:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985124736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390652:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985124736:12345-0@lo:400:kworker.0 Request processed in 28us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319260.390657:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2974 00000100:00080000:3.0:1767319260.390681:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319260.390685:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319260.390740:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985124864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390766:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985124736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390780:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124480 00000100:00100000:3.0:1767319260.390783:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124480 00000100:00100000:3.0:1767319260.390827:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4512 00000100:00100000:3.0:1767319260.390835:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985124480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.390868:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985124480:12345-0@lo:400:kworker.0 Request processed in 36us (418us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319260.390874:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4512 00000100:00080000:3.0:1767319260.390888:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319260.390902:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985124480:0@lo:400:kworker.0 00000100:00080000:3.0:1767319260.390975:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319260.390980:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319260.390983:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985124352:0@lo:38:kworker.0 00000100:00080000:3.0:1767319260.390991:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319260.390994:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319260.390996:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985124608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319260.397290:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985124224:0@lo:400:kworker.0 00000100:00100000:0.0:1767319260.397370:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319260.401247:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124224 00000100:00100000:1.0:1767319260.401262:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124224 00000100:00100000:1.0:1767319260.401395:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4513 00000100:00100000:1.0:1767319260.401406:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985124224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319260.401514:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985124224:12345-0@lo:400:kworker.0 Request processed in 110us (4144us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319260.401522:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4513 00000100:00100000:0.0:1767319260.402619:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985124224:0@lo:400:kworker.0 00000100:00080000:3.0:1767319260.407244:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319261.019442:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319262.044425:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319263.071896:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319263.966267:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985124992:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319263.966388:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319263.966424:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319263.966435:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985125120:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319263.966462:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319263.966520:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985124992 00000100:00100000:2.0:1767319263.966529:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985124992 00000100:00100000:2.0:1767319263.966593:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2136 00000100:00100000:1.0:1767319263.966603:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985125120 00000100:00100000:2.0:1767319263.966603:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985124992:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319263.966607:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985125120 00000100:00100000:1.0:1767319263.966631:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2137 00000100:00100000:1.0:1767319263.966637:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985125120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319263.966645:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:1.0:1767319263.966651:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319263.966685:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985125120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (222us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319263.966686:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985124992:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (297us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319263.966693:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2137 00000100:00100000:2.0:1767319263.966696:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2136 00000100:00100000:1.0:1767319263.966734:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985125120:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319263.966984:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319263.966997:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319263.967023:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985124992:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319264.091489:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319264.426525:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319264.426564:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920550784 offset 0 length 520: 4 00000100:00100000:2.0:1767319264.426609:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319264.426693:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551040 00000100:00100000:3.0:1767319264.426704:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551040 00000100:00100000:3.0:1767319264.426822:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920550912 00000100:00100000:3.0:1767319264.426826:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920550912 00000100:00080000:0.0:1767319264.426839:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319264.426848:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4514 00000100:00100000:3.0:1767319264.426858:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920551040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319264.426872:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4955 00000100:00100000:0.0:1767319264.426877:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920550912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319264.426907:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319264.427012:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920551040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 155us (496us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319264.427020:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4514 00000100:00100000:0.0:1767319264.437747:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920550912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 10847us (11116us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319264.437762:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4955 00000100:00100000:1.0:1767319264.475365:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319239/real 1767319239] req@ffff8ce188f04380 x1853163985119744/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319264 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319264.475410:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188f04380 x1853163985119744/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319264 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319264.475430:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319264.475457:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319264.475461:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985119744:0@lo:250:kworker.0 00000100:00080000:2.0:1767319264.475632:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319264.475638:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319264.475641:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319264.475646:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4488 00000100:00080000:2.0:1767319264.475650:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319264.475656:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319264.475698:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7b80 x1853163985125248/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319264.475721:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319264.475725:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319264.475727:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319264.475731:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4508 00000100:00080000:2.0:1767319264.475734:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319264.475737:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319264.475755:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4380 x1853163985125376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319264.475763:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319264.475766:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319264.475768:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319264.475770:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4508 00000100:00080000:2.0:1767319264.475774:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319264.475777:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319264.475787:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7800 x1853163985125504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319264.475832:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319264.475836:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319264.475838:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319264.475842:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4508 00000100:00080000:2.0:1767319264.475844:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319264.475847:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319264.475862:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7100 x1853163985125760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319264.475878:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319264.475900:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985125632:0@lo:400:kworker.0 00000100:00080000:2.0:1767319264.475941:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319264.475944:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319264.475946:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319264.475949:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4508 00000100:00080000:2.0:1767319264.475951:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319264.475954:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319264.475964:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5c00 x1853163985126016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319264.475967:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319264.476019:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985125888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319264.476069:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319264.476104:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985125888 00000100:00100000:2.0:1767319264.476104:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985125632 00000100:00100000:0.0:1767319264.476112:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985125888 00000100:00100000:2.0:1767319264.476112:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985125632 00000100:00100000:1.0:1767319264.476157:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985125248:0@lo:250:kworker.0 00000100:00100000:0.0:1767319264.476160:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4515 00000400:00000100:1.0:1767319264.476199:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985125248 offset 0 length 520: 4 00000100:00080000:3.0:1767319264.476272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319264.476313:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985125376:0@lo:38:kworker.0 00000100:00100000:1.0:1767319264.476337:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319264.476353:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985125504:0@lo:38:kworker.0 00000100:00100000:1.0:1767319264.476412:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319264.476431:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985125760:0@lo:38:kworker.0 00000100:00100000:1.0:1767319264.476452:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319264.476468:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985126016:0@lo:38:kworker.0 00000100:00100000:1.0:1767319264.476485:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319264.476522:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985125504 00000100:00100000:1.0:1767319264.476526:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985125504 00000100:00100000:1.0:1767319264.476547:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1729 00000100:00100000:1.0:1767319264.476552:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190a43450 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985125504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319264.476570:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319264.476611:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190a43450 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985125504:12345-0@lo:38:kworker.0 Request processed in 57us (198us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319264.476618:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1729 00000100:00080000:1.0:1767319264.476799:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319264.476804:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319264.476807:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985125504:0@lo:38:kworker.0 00000100:00100000:1.0:1767319264.476839:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985125376 00000100:00100000:1.0:1767319264.476843:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985125376 00000100:00100000:1.0:1767319264.476862:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985125760 00000100:00100000:1.0:1767319264.476864:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985125760 00000100:00100000:1.0:1767319264.476872:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126016 00000100:00100000:1.0:1767319264.476874:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126016 00000100:00100000:1.0:1767319264.476894:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4956 00000100:00100000:1.0:1767319264.476898:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985125376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319264.476908:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319264.476953:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985125376:12345-0@lo:38:kworker.0 Request processed in 53us (614us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319264.476959:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4956 00000100:00100000:1.0:1767319264.476972:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4957 00000100:00100000:1.0:1767319264.476975:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985125760:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319264.476983:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319264.477004:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985125760:12345-0@lo:38:kworker.0 Request processed in 28us (552us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319264.477009:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4957 00000100:00100000:1.0:1767319264.477020:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4958 00000100:00100000:1.0:1767319264.477024:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985126016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319264.477033:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319264.477055:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985126016:12345-0@lo:38:kworker.0 Request processed in 30us (569us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319264.477059:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4958 00000100:00080000:1.0:1767319264.477083:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319264.477087:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319264.477090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985125376:0@lo:38:kworker.0 00000100:00080000:1.0:1767319264.477097:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319264.477100:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319264.477103:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985125760:0@lo:38:kworker.0 00000100:00080000:1.0:1767319264.477110:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319264.477112:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319264.477115:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985126016:0@lo:38:kworker.0 00000100:00100000:2.0:1767319264.478961:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4516 00000100:00100000:2.0:1767319264.478973:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985125632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319264.479081:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985125632:12345-0@lo:400:kworker.0 Request processed in 110us (3113us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319264.479087:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4516 00000100:00100000:2.0:1767319264.479189:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985125632:0@lo:400:kworker.0 00000100:00100000:0.0:1767319264.480868:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985125888:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319264.481028:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985125888:12345-0@lo:400:kworker.0 Request processed in 4824us (4957us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319264.481037:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4515 00000100:00100000:2.0:1767319264.483221:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985125888:0@lo:400:kworker.0 00010000:00010000:1.0:1767319265.115433:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319266.140823:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319267.163576:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319268.187397:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319269.083605:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319269.083641:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985126272:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319269.085107:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319269.085230:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126272 00000100:00100000:3.0:1767319269.085240:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126272 00000100:00100000:2.0:1767319269.085374:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985126144:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319269.085433:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319269.091309:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319269.098882:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126144 00000100:00100000:1.0:1767319269.098897:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126144 00000100:00100000:1.0:1767319269.098966:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2138 00000100:00100000:1.0:1767319269.098976:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb4850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985126272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319269.099010:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319269.099080:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb4850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985126272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (13980us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319269.099087:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2138 00000100:00100000:1.0:1767319269.099340:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2139 00000100:00100000:1.0:1767319269.099347:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985126144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319269.099366:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319269.099398:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985126144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 51us (13964us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319269.099404:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2139 00000100:00100000:1.0:1767319269.099458:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985126272:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319269.099616:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319269.099644:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985126144:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319269.211543:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319269.545899:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319269.545957:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319269.546060:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551296 00000100:00100000:2.0:1767319269.546071:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551296 00000100:00100000:2.0:1767319269.546153:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551168 00000100:00100000:2.0:1767319269.546157:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551168 00000100:00100000:2.0:1767319269.546206:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4517 00000100:00100000:2.0:1767319269.546219:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920551296:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319269.546283:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920551296:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 69us (395us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319269.546288:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4517 00000100:00100000:2.0:1767319269.546304:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4959 00000100:00100000:2.0:1767319269.546306:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920551168:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319269.546322:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319269.546350:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920551168:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (394us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319269.546354:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4959 00000100:00080000:0.0:1767319269.546873:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319269.596414:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319269.596424:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319269.596428:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319269.596432:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319269.596437:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319269.596444:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4512 00000100:00080000:3.0:1767319269.596450:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319269.596456:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319269.596494:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adf80 x1853163985126400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319269.596525:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319269.596529:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319269.596531:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319269.596535:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4512 00000100:00080000:3.0:1767319269.596537:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319269.596540:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319269.596562:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ace00 x1853163985126528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319269.596607:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319269.596611:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319269.596613:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319269.596617:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4512 00000100:00080000:3.0:1767319269.596619:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319269.596622:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319269.596633:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad500 x1853163985126784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319269.596659:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319269.596662:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319269.596663:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319269.596667:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4512 00000100:00080000:3.0:1767319269.596669:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319269.596672:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319269.596683:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adc00 x1853163985127040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319269.596711:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985126912:0@lo:400:kworker.0 00000100:00100000:3.0:1767319269.596808:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319269.597062:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985126656:0@lo:400:kworker.0 00000100:00100000:2.0:1767319269.597106:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319269.597423:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126912 00000100:00100000:1.0:1767319269.597431:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126912 00000100:00100000:1.0:1767319269.597496:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126656 00000100:00100000:1.0:1767319269.597499:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126656 00000100:00100000:1.0:1767319269.597524:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4518 00000100:00100000:1.0:1767319269.597534:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985126912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319269.597600:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985126912:12345-0@lo:400:kworker.0 Request processed in 67us (791us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319269.597608:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4518 00000100:00100000:1.0:1767319269.597625:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4519 00000100:00100000:1.0:1767319269.597632:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985126656:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319269.597664:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985126656:12345-0@lo:400:kworker.0 Request processed in 32us (557us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319269.597669:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4519 00000100:00100000:1.0:1767319269.597695:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985126400:0@lo:38:kworker.0 00000100:00100000:1.0:1767319269.597721:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319269.597742:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985126528:0@lo:38:kworker.0 00000100:00100000:1.0:1767319269.597767:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319269.597799:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985126784:0@lo:38:kworker.0 00000100:00100000:1.0:1767319269.597822:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319269.597976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319269.598015:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985127040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319269.598040:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319269.598153:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126400 00000100:00100000:0.0:1767319269.598159:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126400 00000100:00100000:0.0:1767319269.598209:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126784 00000100:00100000:0.0:1767319269.598213:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126784 00000100:00100000:0.0:1767319269.598230:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4960 00000100:00100000:0.0:1767319269.598235:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985126400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319269.598255:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319269.598292:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985126400:12345-0@lo:38:kworker.0 Request processed in 56us (570us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319269.598299:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4960 00000100:00100000:0.0:1767319269.598319:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4961 00000100:00100000:0.0:1767319269.598324:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985126784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319269.598333:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319269.598354:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985126784:12345-0@lo:38:kworker.0 Request processed in 29us (531us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319269.598360:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4961 00000100:00100000:0.0:1767319269.598405:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985126528 00000100:00100000:0.0:1767319269.598409:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985126528 00000100:00100000:0.0:1767319269.598446:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1730 00000100:00100000:0.0:1767319269.598451:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985126528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319269.598461:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319269.598485:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985126528:12345-0@lo:38:kworker.0 Request processed in 33us (718us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319269.598495:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1730 00000100:00100000:0.0:1767319269.598556:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985126656:0@lo:400:kworker.0 00000100:00100000:0.0:1767319269.598600:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985126912:0@lo:400:kworker.0 00000100:00080000:1.0:1767319269.603939:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319269.603952:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319269.603957:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985126400:0@lo:38:kworker.0 00000100:00080000:1.0:1767319269.603970:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319269.603974:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319269.603976:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985126528:0@lo:38:kworker.0 00000100:00080000:1.0:1767319269.603986:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319269.603989:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319269.603992:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985126784:0@lo:38:kworker.0 00000100:00100000:3.0:1767319269.608182:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127040 00000100:00100000:3.0:1767319269.608207:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127040 00000100:00100000:3.0:1767319269.608284:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4962 00000100:00100000:3.0:1767319269.608293:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985127040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319269.608318:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319269.608414:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985127040:12345-0@lo:38:kworker.0 Request processed in 120us (10372us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319269.608486:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4962 00000100:00080000:1.0:1767319269.608719:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319269.608726:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319269.608730:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985127040:0@lo:38:kworker.0 00010000:00010000:1.0:1767319270.235422:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319271.259521:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319272.285623:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319273.307345:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319274.203425:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319274.204063:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985127296:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319274.204261:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319274.204918:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127296 00000100:00100000:2.0:1767319274.204927:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127296 00000100:00100000:2.0:1767319274.205459:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2140 00000100:00100000:1.0:1767319274.205542:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985127168:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319274.205592:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319274.205909:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985127296:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319274.205959:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319274.206070:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127168 00000100:00100000:3.0:1767319274.206076:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127168 00000100:00080000:0.0:1767319274.206241:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319274.206246:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319274.207459:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985127296:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1985us (3191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319274.207472:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2140 00000100:00100000:3.0:1767319274.207631:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2141 00000100:00100000:3.0:1767319274.207641:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985127168:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319274.207680:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319274.207741:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985127168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (2147us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319274.207747:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2141 00000100:00100000:3.0:1767319274.207916:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985127168:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319274.208264:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985127296:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319274.332888:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319274.665554:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319274.665660:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551552 00000100:00100000:3.0:1767319274.665671:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551552 00000100:00100000:3.0:1767319274.665752:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4520 00000100:00100000:3.0:1767319274.665764:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920551552:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:2.0:1767319274.665787:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319274.665867:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920551552:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (322us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319274.666024:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319274.666089:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551424 00000100:00100000:0.0:1767319274.666094:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551424 00000100:00100000:3.0:1767319274.666941:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4520 00000100:00100000:0.0:1767319274.668327:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4963 00000100:00100000:0.0:1767319274.668344:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920551424:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319274.668367:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319274.668479:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920551424:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 136us (2455us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319274.668487:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4963 00000100:00080000:2.0:1767319274.715325:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319274.715336:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319274.715340:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319274.715344:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319274.715348:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319274.715357:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4517 00000100:00080000:2.0:1767319274.715362:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319274.715369:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319274.715408:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba5c00 x1853163985127424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319274.715433:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319274.715436:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319274.715438:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319274.715441:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4517 00000100:00080000:2.0:1767319274.715444:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319274.715447:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319274.715459:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba4700 x1853163985127552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319274.715501:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319274.715504:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319274.715506:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319274.715553:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4517 00000100:00080000:2.0:1767319274.715556:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319274.715559:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319274.715574:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba7480 x1853163985127808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319274.715597:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319274.715601:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319274.715603:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319274.715606:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4517 00000100:00080000:2.0:1767319274.715609:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319274.715612:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319274.715622:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba5880 x1853163985128064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319274.715749:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985127936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319274.715750:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985127424:0@lo:38:kworker.0 00000100:00100000:0.0:1767319274.715846:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985127680:0@lo:400:kworker.0 00000100:00100000:3.0:1767319274.723453:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319274.723760:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127936 00000100:00100000:2.0:1767319274.723769:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127936 00000100:00100000:2.0:1767319274.723837:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4521 00000100:00100000:2.0:1767319274.723846:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985127936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319274.723912:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985127936:12345-0@lo:400:kworker.0 Request processed in 67us (466us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319274.723919:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4521 00000100:00080000:3.0:1767319274.726382:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319274.726460:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985127936:0@lo:400:kworker.0 00000100:00100000:0.0:1767319274.727067:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319274.727756:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127680 00000100:00100000:3.0:1767319274.727764:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127680 00000100:00100000:3.0:1767319274.727833:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4522 00000100:00100000:3.0:1767319274.727842:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985127680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319274.727915:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985127680:12345-0@lo:400:kworker.0 Request processed in 74us (849us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319274.727921:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4522 00000100:00080000:0.0:1767319274.728054:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319274.728082:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985127680:0@lo:400:kworker.0 00000100:00100000:1.0:1767319274.731953:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319274.732029:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985127552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.732120:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319274.732143:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985127808:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.732190:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319274.732218:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985128064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.732239:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319274.732290:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127424 00000100:00100000:1.0:1767319274.732300:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127424 00000100:00100000:1.0:1767319274.732337:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127808 00000100:00100000:1.0:1767319274.732341:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127808 00000100:00100000:1.0:1767319274.732352:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128064 00000100:00100000:1.0:1767319274.732354:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128064 00000100:00100000:1.0:1767319274.732369:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4964 00000100:00100000:1.0:1767319274.732375:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985127424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319274.732396:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319274.732438:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985127424:12345-0@lo:38:kworker.0 Request processed in 61us (493us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319274.732445:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4964 00000100:00100000:1.0:1767319274.732462:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4965 00000100:00100000:1.0:1767319274.732466:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985127808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319274.732475:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319274.732500:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985127808:12345-0@lo:38:kworker.0 Request processed in 33us (309us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319274.732506:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4965 00000100:00100000:1.0:1767319274.732517:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4966 00000100:00100000:1.0:1767319274.732521:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985128064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319274.732530:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319274.732569:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985128064:12345-0@lo:38:kworker.0 Request processed in 46us (329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319274.732578:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4966 00000100:00080000:1.0:1767319274.732624:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.732630:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.732633:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985127424:0@lo:38:kworker.0 00000100:00080000:1.0:1767319274.732641:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.732644:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.732647:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985127808:0@lo:38:kworker.0 00000100:00080000:1.0:1767319274.732653:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.732656:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.732658:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985128064:0@lo:38:kworker.0 00000100:00100000:0.0:1767319274.732841:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985127552 00000100:00100000:0.0:1767319274.732846:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985127552 00000100:00100000:0.0:1767319274.732885:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1731 00000100:00100000:0.0:1767319274.732890:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985127552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319274.732901:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319274.732948:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985127552:12345-0@lo:38:kworker.0 Request processed in 56us (827us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319274.732954:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1731 00000100:00080000:1.0:1767319274.733124:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.733128:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.733132:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985127552:0@lo:38:kworker.0 00010000:00010000:0.0:1767319275.355386:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319276.379399:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319277.410377:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319278.427668:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319279.324095:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985128192:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319279.324183:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319279.324303:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128192 00000100:00100000:1.0:1767319279.324312:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128192 00000100:00100000:1.0:1767319279.324379:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2142 00000100:00100000:1.0:1767319279.324388:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985128192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319279.324417:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319279.324467:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985128192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319279.324474:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2142 00000100:00100000:1.0:1767319279.324519:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985128192:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319279.324677:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319279.325606:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319279.325645:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985128320:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319279.325689:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319279.325778:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128320 00000100:00100000:2.0:1767319279.325792:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128320 00000100:00100000:2.0:1767319279.325846:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2143 00000100:00100000:2.0:1767319279.325856:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985128320:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319279.325864:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767319279.325879:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319279.325928:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985128320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319279.325935:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2143 00000100:00100000:3.0:1767319279.325986:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985128320:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319279.454754:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319279.786517:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319279.786615:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319279.786694:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551808 00000100:00100000:1.0:1767319279.786705:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551808 00000100:00100000:1.0:1767319279.786789:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551680 00000100:00100000:1.0:1767319279.786793:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551680 00000100:00100000:1.0:1767319279.786815:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4523 00000100:00100000:1.0:1767319279.786825:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920551808:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319279.786898:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920551808:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (392us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319279.786905:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4523 00000100:00100000:1.0:1767319279.786924:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4967 00000100:00100000:1.0:1767319279.786928:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920551680:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319279.786948:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319279.786973:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920551680:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (359us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319279.786980:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4967 00000100:00080000:0.0:1767319279.787220:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319279.835920:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319279.835928:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319279.835931:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319279.835934:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319279.835938:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319279.835945:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4522 00000100:00080000:3.0:1767319279.835950:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319279.835956:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319279.835989:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac000 x1853163985128448/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319279.836015:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319279.836018:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319279.836020:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319279.836024:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4522 00000100:00080000:3.0:1767319279.836026:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319279.836030:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319279.836044:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad880 x1853163985128576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319279.836071:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319279.836075:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319279.836077:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319279.836080:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4522 00000100:00080000:3.0:1767319279.836082:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319279.836086:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319279.836096:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae680 x1853163985128832/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319279.836120:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319279.836124:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319279.836225:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319279.836229:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4522 00000100:00080000:3.0:1767319279.836231:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319279.836234:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319279.836249:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886aca80 x1853163985129088/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319279.836287:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985129216:0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.836689:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985128448:0@lo:38:kworker.0 00000100:00100000:1.0:1767319279.836885:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985128704:0@lo:400:kworker.0 00000100:00100000:0.0:1767319279.837624:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985129344:0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.839837:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319279.840081:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985128576:0@lo:38:kworker.0 00000100:00100000:2.0:1767319279.840115:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319279.840144:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985128832:0@lo:38:kworker.0 00000100:00100000:2.0:1767319279.840182:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319279.840202:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985129088:0@lo:38:kworker.0 00000100:00100000:2.0:1767319279.840222:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319279.840275:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128576 00000100:00100000:2.0:1767319279.840284:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128576 00000100:00100000:2.0:1767319279.840319:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1732 00000100:00100000:2.0:1767319279.840326:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985128576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319279.840635:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319279.840680:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985128576:12345-0@lo:38:kworker.0 Request processed in 352us (563us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319279.840688:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1732 00000100:00100000:2.0:1767319279.840710:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128448 00000100:00100000:2.0:1767319279.840714:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128448 00000100:00100000:2.0:1767319279.840730:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128832 00000100:00100000:2.0:1767319279.840733:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128832 00000100:00100000:2.0:1767319279.840743:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129088 00000100:00100000:2.0:1767319279.840745:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129088 00000100:00100000:2.0:1767319279.840769:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4968 00000100:00100000:2.0:1767319279.840774:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985128448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319279.840784:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319279.840807:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985128448:12345-0@lo:38:kworker.0 Request processed in 32us (980us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319279.840813:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4968 00000100:00100000:2.0:1767319279.840822:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4969 00000100:00100000:2.0:1767319279.840826:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985128832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319279.840834:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319279.840852:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985128832:12345-0@lo:38:kworker.0 Request processed in 25us (670us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319279.841994:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4969 00000100:00100000:2.0:1767319279.842023:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4970 00000100:00100000:2.0:1767319279.842029:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985129088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319279.842046:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319279.842101:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985129088:12345-0@lo:38:kworker.0 Request processed in 71us (1878us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319279.842106:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4970 00000100:00080000:2.0:1767319279.842186:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319279.842191:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319279.842195:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985128448:0@lo:38:kworker.0 00000100:00080000:2.0:1767319279.842207:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319279.842210:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319279.842213:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985128576:0@lo:38:kworker.0 00000100:00080000:2.0:1767319279.842221:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319279.842224:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319279.842227:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985128832:0@lo:38:kworker.0 00000100:00080000:2.0:1767319279.842235:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319279.842238:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319279.842240:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985129088:0@lo:38:kworker.0 00000100:00100000:1.0:1767319279.843597:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319279.843676:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985128960:0@lo:400:kworker.0 00000100:00100000:1.0:1767319279.843706:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319279.844405:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128704 00000100:00100000:2.0:1767319279.844414:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128704 00000100:00100000:2.0:1767319279.844469:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985128960 00000100:00100000:2.0:1767319279.844473:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985128960 00000100:00100000:2.0:1767319279.844491:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4524 00000100:00100000:2.0:1767319279.844498:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985128704:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.844553:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985128704:12345-0@lo:400:kworker.0 Request processed in 55us (962us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319279.844560:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4524 00000100:00100000:2.0:1767319279.844575:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4525 00000100:00100000:2.0:1767319279.844581:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985128960:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.844616:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985128960:12345-0@lo:400:kworker.0 Request processed in 35us (908us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319279.844622:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4525 00000100:00100000:2.0:1767319279.844663:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985128704:0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.844684:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985128960:0@lo:400:kworker.0 00000100:00080000:1.0:1767319279.844737:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319279.845225:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319279.845485:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319279.861875:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129216 00000100:00100000:1.0:1767319279.861891:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129216 00000100:00100000:1.0:1767319279.861975:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129344 00000100:00100000:1.0:1767319279.861979:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129344 00000100:00100000:1.0:1767319279.862004:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2975 00000100:00100000:1.0:1767319279.862013:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985129216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319279.862112:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985129216:12345-0@lo:400:kworker.0 Request processed in 99us (16887us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319279.862120:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2975 00000100:00100000:1.0:1767319279.862137:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2976 00000100:00100000:1.0:1767319279.862143:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985129344:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319279.862196:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985129344:12345-0@lo:400:kworker.0 Request processed in 54us (16711us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319279.862202:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2976 00000100:00100000:1.0:1767319279.862275:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985129344:0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.862406:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985129216:0@lo:400:kworker.0 00010000:00010000:0.0:1767319280.476439:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319281.499629:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319282.523405:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319283.547779:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319284.444812:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319284.444832:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985129472:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319284.444901:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319284.445064:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129472 00000100:00100000:0.0:1767319284.445072:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129472 00000100:00100000:0.0:1767319284.445134:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2144 00000100:00100000:0.0:1767319284.445144:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985129472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319284.445198:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319284.445244:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985129472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 101us (342us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319284.445251:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2144 00000100:00080000:3.0:1767319284.445672:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319284.445708:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985129472:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319284.445857:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319284.445868:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985129600:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319284.445900:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319284.447636:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129600 00000100:00100000:0.0:1767319284.447649:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129600 00000100:00100000:0.0:1767319284.447731:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2145 00000100:00100000:0.0:1767319284.447742:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985129600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319284.447773:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319284.447851:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985129600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 112us (1949us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319284.447858:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2145 00000100:00100000:2.0:1767319284.447975:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985129600:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319284.448731:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319284.578474:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319284.906307:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319284.906417:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920552064 00000100:00100000:0.0:1767319284.906428:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920552064 00000100:00100000:0.0:1767319284.906501:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4526 00000100:00100000:0.0:1767319284.906546:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920552064:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319284.906630:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920552064:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (335us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319284.906638:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4526 00000100:00080000:3.0:1767319284.907482:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319284.907822:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319284.907958:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920551936 00000100:00100000:2.0:1767319284.907966:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920551936 00000100:00100000:2.0:1767319284.908010:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4971 00000100:00100000:2.0:1767319284.908017:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920551936:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319284.908039:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319284.908093:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920551936:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 74us (273us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319284.908100:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4971 00000100:00080000:3.0:1767319284.956126:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.956136:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319284.956140:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.956143:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.956147:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.956155:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.956159:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.956185:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319284.956224:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985129728/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319284.956253:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.956257:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.956259:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.956262:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.956265:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.956268:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319284.956283:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985129856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319284.956336:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.956339:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.956341:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.956344:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.956347:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.956350:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319284.956362:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae300 x1853163985130112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319284.956388:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.956391:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.956393:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.956397:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.956399:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.956402:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319284.956414:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac380 x1853163985130368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319284.956453:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985130240:0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.956966:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985130624:0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.957006:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319284.957075:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319284.957082:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985129984:0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.957112:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985130496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.957136:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319284.957215:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985130624 00000100:00100000:1.0:1767319284.957225:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985130624 00000100:00100000:1.0:1767319284.957273:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985130496 00000100:00100000:1.0:1767319284.957276:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985130496 00000100:00100000:1.0:1767319284.957299:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2977 00000100:00100000:1.0:1767319284.957308:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985130624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.957360:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985130624:12345-0@lo:400:kworker.0 Request processed in 52us (353us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319284.957381:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2977 00000100:00100000:1.0:1767319284.957397:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2978 00000100:00100000:1.0:1767319284.957403:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985130496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.957434:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985130496:12345-0@lo:400:kworker.0 Request processed in 31us (298us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319284.957440:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2978 00000100:00100000:1.0:1767319284.957473:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985130496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319284.958113:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319284.958362:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985129728:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.958403:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319284.961429:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985129856:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.961486:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319284.961524:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985130112:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.961558:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319284.964407:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129856 00000100:00100000:1.0:1767319284.964422:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129856 00000100:00100000:1.0:1767319284.964700:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1733 00000100:00100000:1.0:1767319284.964710:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985129856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319284.964737:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319284.964838:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985129856:12345-0@lo:38:kworker.0 Request processed in 127us (3349us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319284.964845:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1733 00000100:00100000:0.0:1767319284.965660:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985130240 00000100:00100000:0.0:1767319284.965670:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985130240 00000100:00100000:0.0:1767319284.965723:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129728 00000100:00100000:0.0:1767319284.965725:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129728 00000100:00100000:0.0:1767319284.965739:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4527 00000100:00100000:0.0:1767319284.965744:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985130240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319284.965827:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985130240:12345-0@lo:400:kworker.0 Request processed in 73us (7709us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319284.965834:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4527 00000100:00100000:0.0:1767319284.965850:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4972 00000100:00100000:0.0:1767319284.966451:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985129728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319284.966470:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319284.966526:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985129728:12345-0@lo:38:kworker.0 Request processed in 77us (8121us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319284.966541:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4972 00000100:00100000:0.0:1767319284.966613:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985130240:0@lo:400:kworker.0 00000100:00080000:1.0:1767319284.967500:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319284.967626:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985130368:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.967672:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319284.967710:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.967715:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.967719:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985129728:0@lo:38:kworker.0 00000100:00080000:2.0:1767319284.967731:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.967735:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.967738:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985129856:0@lo:38:kworker.0 00000100:00100000:1.0:1767319284.967746:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319284.967785:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985130624:0@lo:400:kworker.0 00000100:00100000:0.0:1767319284.967989:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985130112 00000100:00100000:0.0:1767319284.967996:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985130112 00000100:00100000:0.0:1767319284.968031:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4973 00000100:00100000:0.0:1767319284.968037:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985130112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319284.968055:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319284.968094:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985130112:12345-0@lo:38:kworker.0 Request processed in 55us (6534us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319284.968101:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4973 00000100:00100000:0.0:1767319284.968114:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985130368 00000100:00100000:0.0:1767319284.968117:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985130368 00000100:00100000:0.0:1767319284.968131:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985129984 00000100:00100000:0.0:1767319284.968134:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985129984 00000100:00100000:0.0:1767319284.968183:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4528 00000100:00100000:0.0:1767319284.968189:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985129984:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319284.968201:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319284.968211:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4974 00000100:00100000:1.0:1767319284.968216:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985130368:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319284.968226:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985129984:12345-0@lo:400:kworker.0 Request processed in 38us (480us total) trans 0 rc 0/0 00010000:00020000:1.0:1767319284.968226:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319284.968232:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4528 00000100:00100000:1.0:1767319284.968249:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985130368:12345-0@lo:38:kworker.0 Request processed in 32us (577us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319284.968255:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4974 00000100:00100000:0.0:1767319284.968275:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985129984:0@lo:400:kworker.0 00000100:00080000:2.0:1767319284.968350:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.968355:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.968358:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985130112:0@lo:38:kworker.0 00000100:00080000:2.0:1767319284.968365:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.968368:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.968371:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985130368:0@lo:38:kworker.0 00010000:00010000:2.0:1767319285.595408:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319286.619402:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319287.644405:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319288.669690:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319289.052333:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319264/real 1767319264] req@ffff8ce1a61f7b80 x1853163985125248/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319289 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319289.052459:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61f7b80 x1853163985125248/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319289 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319289.052481:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319289.052515:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319289.052519:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985125248:0@lo:250:kworker.0 00000100:00080000:0.0:1767319289.054470:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319289.054479:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319289.054483:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319289.054489:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4512 00000100:00080000:0.0:1767319289.054493:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319289.054500:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319289.054536:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4e00 x1853163985130752/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319289.054567:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319289.054570:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319289.054573:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319289.054577:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4533 00000100:00080000:0.0:1767319289.054579:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319289.054583:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319289.054599:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5500 x1853163985130880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319289.054608:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319289.054611:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319289.054612:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319289.054615:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4533 00000100:00080000:0.0:1767319289.054618:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319289.054620:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319289.054634:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7b80 x1853163985131008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319289.054677:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319289.054680:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319289.054682:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319289.054685:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4533 00000100:00080000:0.0:1767319289.054687:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319289.054690:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319289.054702:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985131264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319289.054730:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319289.054734:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319289.054737:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319289.054741:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4533 00000100:00080000:0.0:1767319289.054744:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319289.054747:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319289.054759:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7480 x1853163985131520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319289.054760:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319289.054804:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985131136:0@lo:400:kworker.0 00000100:00100000:0.0:1767319289.054860:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985131392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319289.055006:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319289.055020:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985131776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319289.055086:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985130752:0@lo:250:kworker.0 00000100:00100000:3.0:1767319289.055214:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319289.055524:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131136 00000100:00100000:3.0:1767319289.055535:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131136 00000100:00100000:3.0:1767319289.055619:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4529 00000100:00100000:3.0:1767319289.055633:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985131136:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319289.055707:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985131136:12345-0@lo:400:kworker.0 Request processed in 76us (494us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319289.057775:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4529 00000100:00080000:3.0:1767319289.057843:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319289.057922:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985131136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319289.058034:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985131648:0@lo:400:kworker.0 00000100:00100000:3.0:1767319289.058127:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319289.058229:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131648 00000100:00100000:3.0:1767319289.058237:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131648 00000100:00100000:0.0:1767319289.058461:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319289.058606:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131392 00000100:00100000:0.0:1767319289.058616:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131392 00000100:00100000:0.0:1767319289.058673:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4530 00000100:00100000:0.0:1767319289.058681:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985131392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319289.058742:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985131392:12345-0@lo:400:kworker.0 Request processed in 62us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319289.058749:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4530 00000100:00100000:3.0:1767319289.058964:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2979 00000100:00100000:3.0:1767319289.058979:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985131648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319289.060641:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985131648:12345-0@lo:400:kworker.0 Request processed in 1654us (2501us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319289.060660:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2979 00000100:00080000:3.0:1767319289.060710:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319289.060718:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319289.060770:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985131648:0@lo:400:kworker.0 00000400:00000100:2.0:1767319289.062208:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985130752 offset 0 length 520: 4 00000100:00100000:2.0:1767319289.062267:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985130880:0@lo:38:kworker.0 00000100:00100000:2.0:1767319289.062333:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319289.062364:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985131008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319289.062395:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319289.062445:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131008 00000100:00100000:2.0:1767319289.062454:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131008 00000100:00100000:2.0:1767319289.062492:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1734 00000100:00100000:2.0:1767319289.062499:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985131008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319289.062519:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319289.062556:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985131008:12345-0@lo:38:kworker.0 Request processed in 56us (159us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319289.062563:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1734 00000100:00080000:2.0:1767319289.062606:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319289.062612:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319289.062616:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985131008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319289.062623:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985131264:0@lo:38:kworker.0 00000100:00100000:2.0:1767319289.062645:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319289.062671:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985131520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319289.062689:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319289.062708:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985130880 00000100:00100000:2.0:1767319289.062712:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985130880 00000100:00100000:2.0:1767319289.062736:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131264 00000100:00100000:2.0:1767319289.062739:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131264 00000100:00100000:2.0:1767319289.062751:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131520 00000100:00100000:2.0:1767319289.062753:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131520 00000100:00100000:2.0:1767319289.062765:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4975 00000100:00100000:2.0:1767319289.062769:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985130880:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319289.062791:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319289.062814:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985130880:12345-0@lo:38:kworker.0 Request processed in 44us (481us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319289.062819:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4975 00000100:00100000:2.0:1767319289.062829:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4976 00000100:00100000:2.0:1767319289.062832:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985131264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319289.062839:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319289.062855:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985131264:12345-0@lo:38:kworker.0 Request processed in 22us (210us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319289.062860:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4976 00000100:00100000:2.0:1767319289.062868:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4977 00000100:00100000:2.0:1767319289.062872:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985131520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319289.062879:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319289.062896:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985131520:12345-0@lo:38:kworker.0 Request processed in 23us (207us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319289.062904:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4977 00000100:00080000:2.0:1767319289.062930:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319289.062933:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319289.062936:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985131520:0@lo:38:kworker.0 00000100:00080000:2.0:1767319289.062961:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319289.062965:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319289.062968:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985130880:0@lo:38:kworker.0 00000100:00080000:2.0:1767319289.062974:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319289.062977:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319289.062980:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985131264:0@lo:38:kworker.0 00000100:00100000:0.0:1767319289.065710:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985131392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319289.066915:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319289.072933:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131776 00000100:00100000:3.0:1767319289.072950:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131776 00000100:00080000:0.0:1767319289.073135:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319289.073567:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2980 00000100:00100000:3.0:1767319289.073578:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985131776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319289.073669:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985131776:12345-0@lo:400:kworker.0 Request processed in 93us (6756us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319289.073677:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2980 00000100:00100000:0.0:1767319289.073757:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985131776:0@lo:400:kworker.0 00000100:00100000:0.0:1767319289.563512:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985131904:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319289.563583:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319289.564629:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985132032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319289.564677:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319289.564768:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985131904 00000100:00100000:0.0:1767319289.564776:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985131904 00000100:00100000:0.0:1767319289.564828:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132032 00000100:00100000:0.0:1767319289.564831:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132032 00000100:00100000:0.0:1767319289.564853:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2146 00000100:00100000:0.0:1767319289.564861:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985131904:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319289.564891:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319289.564937:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985131904:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (1353us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319289.564944:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2146 00000100:00100000:0.0:1767319289.564960:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2147 00000100:00100000:0.0:1767319289.564966:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985132032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319289.564981:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319289.565010:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985132032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (333us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319289.565016:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2147 00000100:00100000:0.0:1767319289.565055:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985131904:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319289.565085:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985132032:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319289.699389:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319290.026966:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767319290.027000:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920552192 offset 0 length 520: 4 00000100:00100000:1.0:1767319290.027038:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319290.027205:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920552448 00000100:00100000:3.0:1767319290.027216:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920552448 00000100:00100000:3.0:1767319290.027290:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920552320 00000100:00100000:3.0:1767319290.027293:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920552320 00000100:00100000:3.0:1767319290.027316:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4531 00000100:00100000:3.0:1767319290.027328:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920552448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319290.027462:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920552448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 135us (503us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319290.027470:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4531 00000100:00080000:0.0:1767319290.027591:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319290.027625:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4978 00000100:00100000:0.0:1767319290.027631:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920552320:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319290.027651:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319290.027679:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920552320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (642us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319290.027686:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4978 00010000:00010000:2.0:1767319290.716723:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319291.740722:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319292.763936:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319293.787474:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319294.172412:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319294.172421:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319294.172424:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319294.172428:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319294.172432:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319294.172440:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4537 00000100:00080000:3.0:1767319294.172445:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319294.172452:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319294.172490:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac380 x1853163985132160/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319294.172518:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319294.172522:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319294.172524:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319294.172527:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4537 00000100:00080000:3.0:1767319294.172529:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319294.172533:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319294.172546:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae300 x1853163985132288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319294.172574:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319294.172578:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319294.172580:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319294.172583:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4537 00000100:00080000:3.0:1767319294.172585:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319294.172588:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319294.172597:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886aca80 x1853163985132544/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319294.172620:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319294.172623:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319294.172625:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319294.172630:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4537 00000100:00100000:1.0:1767319294.172631:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985132160:0@lo:38:kworker.0 00000100:00080000:3.0:1767319294.172632:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319294.172638:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319294.172651:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985132800/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319294.172710:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319294.172808:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985133056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.172844:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319294.172867:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985132288:0@lo:38:kworker.0 00000100:00100000:1.0:1767319294.172896:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319294.172919:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985132544:0@lo:38:kworker.0 00000100:00100000:1.0:1767319294.172938:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319294.172959:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985132800:0@lo:38:kworker.0 00000100:00100000:1.0:1767319294.172978:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319294.173008:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132160 00000100:00100000:1.0:1767319294.173015:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132160 00000100:00100000:1.0:1767319294.173046:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132544 00000100:00100000:1.0:1767319294.173049:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132544 00000100:00100000:1.0:1767319294.173062:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132800 00000100:00100000:1.0:1767319294.173065:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132800 00000100:00100000:0.0:1767319294.173081:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4979 00000100:00100000:1.0:1767319294.173091:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4980 00000100:00100000:0.0:1767319294.173093:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985132160:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319294.173101:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985132544:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319294.173122:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319294.173189:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985132160:12345-0@lo:38:kworker.0 Request processed in 98us (479us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319294.173197:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4979 00000100:00100000:0.0:1767319294.173214:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4981 00000100:00100000:0.0:1767319294.173218:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985132800:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319294.173257:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319294.173266:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985132416:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173289:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319294.173310:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985132928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173332:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319294.173372:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319294.173378:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985132672:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173399:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319294.173425:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133056 00000100:00100000:3.0:1767319294.173429:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133056 00000100:00100000:3.0:1767319294.173472:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132928 00000100:00100000:3.0:1767319294.173476:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132928 00000100:00100000:3.0:1767319294.173496:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2981 00000100:00100000:3.0:1767319294.173502:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985133056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173545:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985133056:12345-0@lo:400:kworker.0 Request processed in 42us (701us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319294.173551:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2981 00000100:00100000:3.0:1767319294.173564:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2982 00000100:00100000:3.0:1767319294.173570:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985132928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173599:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985132928:12345-0@lo:400:kworker.0 Request processed in 30us (266us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319294.173605:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2982 00000100:00080000:3.0:1767319294.173620:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319294.173624:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319294.173657:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985132928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173683:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985133056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173709:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132416 00000100:00100000:3.0:1767319294.173712:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132416 00000100:00100000:3.0:1767319294.173733:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132672 00000100:00100000:3.0:1767319294.173736:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132672 00000100:00100000:3.0:1767319294.173748:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4532 00000100:00100000:3.0:1767319294.173753:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985132416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173786:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985132416:12345-0@lo:400:kworker.0 Request processed in 33us (496us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319294.173791:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4532 00000100:00100000:3.0:1767319294.173802:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4533 00000100:00100000:3.0:1767319294.173808:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985132672:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.173837:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985132672:12345-0@lo:400:kworker.0 Request processed in 30us (437us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319294.173842:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4533 00000100:00080000:3.0:1767319294.173856:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319294.173878:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985132288 00000100:00100000:2.0:1767319294.173884:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985132288 00000100:00100000:3.0:1767319294.173887:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985132672:0@lo:400:kworker.0 00000100:00100000:2.0:1767319294.173914:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1735 00000100:00100000:3.0:1767319294.173917:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985132416:0@lo:400:kworker.0 00000100:00100000:2.0:1767319294.173924:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985132288:12345-0@lo:38:kworker.0 00000100:00080000:3.0:1767319294.173945:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00020000:1.0:1767319294.177420:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319294.177503:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985132544:12345-0@lo:38:kworker.0 Request processed in 4400us (4561us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319294.177515:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4980 00000100:00080000:3.0:1767319294.178664:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319294.178672:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985132160:0@lo:38:kworker.0 00000100:00080000:3.0:1767319294.178708:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319294.178713:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319294.178717:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985132544:0@lo:38:kworker.0 00010000:00020000:0.0:1767319294.185512:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319294.185627:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985132800:12345-0@lo:38:kworker.0 Request processed in 12403us (12644us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319294.185640:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4981 00000100:00080000:3.0:1767319294.185819:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319294.185825:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319294.185830:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985132800:0@lo:38:kworker.0 00010000:00020000:2.0:1767319294.189235:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319294.189350:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985132288:12345-0@lo:38:kworker.0 Request processed in 15414us (16439us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319294.189362:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1735 00000100:00080000:3.0:1767319294.191322:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319294.191333:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319294.191347:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985132288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319294.684092:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985133312:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319294.684146:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319294.684159:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985133184:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319294.684185:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319294.684213:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319294.684244:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133312 00000100:00100000:0.0:1767319294.684252:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133312 00000100:00100000:2.0:1767319294.684279:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133184 00000100:00100000:2.0:1767319294.684283:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133184 00000100:00100000:0.0:1767319294.684305:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2148 00000100:00100000:0.0:1767319294.684313:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985133312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319294.684352:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319294.684399:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985133312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (212us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319294.686143:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2149 00000100:00100000:2.0:1767319294.686181:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985133184:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319294.686213:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319294.686286:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985133184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 131us (2071us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319294.686294:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2149 00000100:00100000:2.0:1767319294.686372:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985133184:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319294.686438:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985133312:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319294.688005:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2148 00010000:00010000:1.0:1767319294.817684:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319295.147553:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319295.147684:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319295.147793:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920552576 00000100:00100000:3.0:1767319295.147802:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920552576 00000100:00100000:3.0:1767319295.147857:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4982 00000100:00100000:1.0:1767319295.147860:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920552704 00000100:00100000:1.0:1767319295.147865:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920552704 00000100:00100000:3.0:1767319295.147865:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920552576:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319295.147888:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319295.147953:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4534 00000100:00100000:1.0:1767319295.147961:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920552704:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319295.148034:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920552704:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319295.148042:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4534 00000100:00100000:3.0:1767319295.148080:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920552576:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 214us (537us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319295.148092:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4982 00000100:00080000:0.0:1767319295.148546:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319295.835412:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319296.865514:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319297.883506:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319298.908194:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319299.292710:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319299.292721:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319299.292724:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319299.292728:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319299.292732:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319299.292739:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4542 00000100:00080000:0.0:1767319299.292743:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319299.292749:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319299.292788:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5500 x1853163985133440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319299.292814:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319299.292818:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319299.292820:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319299.292823:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4542 00000100:00080000:0.0:1767319299.292825:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319299.292828:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319299.292840:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7480 x1853163985133568/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319299.292879:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319299.292883:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319299.292885:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319299.292888:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4542 00000100:00080000:0.0:1767319299.292890:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319299.292894:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319299.292905:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6d80 x1853163985133824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319299.292928:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319299.292931:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319299.292933:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319299.292936:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4542 00000100:00080000:0.0:1767319299.292939:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319299.292941:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319299.292955:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61180 x1853163985134080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319299.293041:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985134208:0@lo:400:kworker.0 00000100:00100000:1.0:1767319299.293051:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985133696:0@lo:400:kworker.0 00000100:00100000:3.0:1767319299.293107:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319299.293157:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319299.293193:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319299.293200:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985133952:0@lo:400:kworker.0 00000100:00100000:3.0:1767319299.293229:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985133440:0@lo:38:kworker.0 00000100:00100000:3.0:1767319299.293273:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133696 00000100:00100000:3.0:1767319299.293280:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133696 00000100:00100000:1.0:1767319299.293302:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319299.293309:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985134336:0@lo:400:kworker.0 00000100:00100000:3.0:1767319299.293341:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4535 00000100:00100000:3.0:1767319299.293353:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985133696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319299.293409:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319299.293409:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985133696:12345-0@lo:400:kworker.0 Request processed in 58us (251us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319299.293420:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4535 00000100:00080000:3.0:1767319299.293445:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319299.293448:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985133696:0@lo:400:kworker.0 00000100:00100000:3.0:1767319299.293463:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134208 00000100:00100000:3.0:1767319299.293469:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134208 00000100:00080000:1.0:1767319299.293488:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319299.293501:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134336 00000100:00100000:3.0:1767319299.293505:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134336 00000100:00100000:1.0:1767319299.293516:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2983 00000100:00100000:1.0:1767319299.293526:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985134208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319299.293547:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2984 00000100:00100000:1.0:1767319299.293561:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985134208:12345-0@lo:400:kworker.0 Request processed in 36us (452us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319299.293561:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985134336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319299.293574:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2983 00000100:00080000:1.0:1767319299.293599:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319299.293613:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985134336:12345-0@lo:400:kworker.0 Request processed in 56us (204us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319299.293622:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2984 00000100:00100000:1.0:1767319299.293630:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985134336:0@lo:400:kworker.0 00000100:00100000:3.0:1767319299.293647:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319299.293673:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133952 00000100:00100000:1.0:1767319299.293678:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133952 00000100:00100000:3.0:1767319299.293678:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985134208:0@lo:400:kworker.0 00000100:00100000:3.0:1767319299.293705:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319299.293761:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319299.293948:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985133568:0@lo:38:kworker.0 00000100:00100000:3.0:1767319299.293973:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319299.294003:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985133824:0@lo:38:kworker.0 00000100:00100000:3.0:1767319299.294024:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319299.294076:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133568 00000100:00100000:0.0:1767319299.294080:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133568 00000100:00100000:0.0:1767319299.294103:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1736 00000100:00100000:0.0:1767319299.294107:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985133568:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319299.294126:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319299.294157:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985133568:12345-0@lo:38:kworker.0 Request processed in 49us (183us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319299.294182:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1736 00000100:00100000:1.0:1767319299.296531:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133440 00000100:00100000:1.0:1767319299.296545:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133440 00000100:00100000:1.0:1767319299.296598:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4536 00000100:00100000:1.0:1767319299.296615:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985133952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319299.296706:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985133952:12345-0@lo:400:kworker.0 Request processed in 100us (3059us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319299.296713:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4536 00000100:00100000:1.0:1767319299.296731:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4983 00000100:00100000:1.0:1767319299.296735:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985133440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319299.296754:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319299.296780:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985133440:12345-0@lo:38:kworker.0 Request processed in 44us (3075us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319299.296785:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4983 00000100:00100000:1.0:1767319299.296826:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985133952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319299.301253:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985133824 00000100:00100000:1.0:1767319299.301268:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985133824 00000100:00100000:3.0:1767319299.301272:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985134080:0@lo:38:kworker.0 00000100:00100000:1.0:1767319299.301319:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4984 00000100:00100000:1.0:1767319299.301329:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985133824:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319299.301342:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319299.301398:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00020000:1.0:1767319299.301403:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319299.301407:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319299.301413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985133440:0@lo:38:kworker.0 00000100:00080000:3.0:1767319299.301426:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319299.301430:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319299.301434:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985133568:0@lo:38:kworker.0 00000100:00100000:1.0:1767319299.301454:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985133824:12345-0@lo:38:kworker.0 Request processed in 124us (7427us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767319299.301461:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319299.301463:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4984 00000100:00080000:3.0:1767319299.301468:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319299.301472:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985133824:0@lo:38:kworker.0 00000100:00100000:1.0:1767319299.301481:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134080 00000100:00100000:1.0:1767319299.301485:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134080 00000100:00100000:1.0:1767319299.301507:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4985 00000100:00100000:1.0:1767319299.301511:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985134080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319299.301520:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319299.301644:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985134080:12345-0@lo:38:kworker.0 Request processed in 131us (302us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319299.301651:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4985 00000100:00080000:3.0:1767319299.302343:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319299.302349:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319299.302353:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985134080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319299.803372:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319299.803396:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985134464:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319299.803463:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319299.803466:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319299.803479:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985134592:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319299.803527:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319299.803787:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134464 00000100:00100000:2.0:1767319299.803796:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134464 00000100:00100000:0.0:1767319299.803807:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134592 00000100:00100000:0.0:1767319299.803813:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134592 00000100:00100000:2.0:1767319299.803893:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2150 00000100:00100000:2.0:1767319299.803904:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985134464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319299.803933:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319299.803981:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985134464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (519us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319299.803988:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2150 00000100:00100000:2.0:1767319299.804005:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2151 00000100:00100000:2.0:1767319299.804011:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985134592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319299.804025:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319299.804053:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985134592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 42us (534us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319299.804062:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2151 00000100:00100000:3.0:1767319299.804132:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985134464:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319299.804211:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985134592:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319299.940612:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319300.269680:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319300.269725:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319300.270778:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920552832 00000100:00100000:2.0:1767319300.270790:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920552832 00000100:00100000:2.0:1767319300.270838:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920552960 00000100:00100000:2.0:1767319300.270841:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920552960 00000100:00100000:2.0:1767319300.270900:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4537 00000100:00100000:2.0:1767319300.270910:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920552960:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319300.270988:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920552960:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (1262us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319300.270995:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4537 00000100:00100000:1.0:1767319300.273460:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4986 00000100:00100000:1.0:1767319300.273470:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920552832:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319300.273491:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319300.273574:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920552832:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 102us (3904us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319300.273582:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4986 00000100:00080000:1.0:1767319300.273650:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319300.955529:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319301.981943:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319303.004547:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319304.029052:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319304.412090:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319304.412100:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319304.412103:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319304.412107:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319304.412111:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319304.412119:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4547 00000100:00080000:0.0:1767319304.412124:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319304.412131:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319304.412198:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60700 x1853163985134720/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319304.412227:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319304.412230:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319304.412232:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319304.412235:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4547 00000100:00080000:0.0:1767319304.412237:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319304.412241:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319304.412254:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163985134848/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319304.412298:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319304.412302:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319304.412304:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319304.412308:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4547 00000100:00080000:0.0:1767319304.412310:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319304.412313:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319304.412326:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63800 x1853163985135104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319304.412352:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319304.412355:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1767319304.412356:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1767319304.412358:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319304.412365:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4547 00000100:00080000:0.0:1767319304.412368:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319304.412372:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319304.412377:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985134976:0@lo:400:kworker.0 00000100:00100000:0.0:1767319304.412387:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61180 x1853163985135360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319304.412460:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319304.412468:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319304.412480:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985135616:0@lo:400:kworker.0 00000100:00100000:1.0:1767319304.412529:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319304.412570:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985135616 00000100:00100000:1.0:1767319304.412578:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985135616 00000100:00100000:1.0:1767319304.412635:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2985 00000100:00100000:1.0:1767319304.412645:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985135616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319304.412694:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985135616:12345-0@lo:400:kworker.0 Request processed in 51us (165us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319304.412701:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2985 00000100:00100000:1.0:1767319304.412738:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985135616:0@lo:400:kworker.0 00000100:00080000:2.0:1767319304.412980:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319304.413034:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134976 00000100:00100000:0.0:1767319304.413039:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134976 00000100:00100000:0.0:1767319304.413082:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4538 00000100:00100000:0.0:1767319304.413090:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985134976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319304.413128:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985134976:12345-0@lo:400:kworker.0 Request processed in 39us (668us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319304.413134:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4538 00000100:00080000:2.0:1767319304.413212:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319304.413233:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985134976:0@lo:400:kworker.0 00000100:00100000:3.0:1767319304.414295:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985135488:0@lo:400:kworker.0 00000100:00100000:3.0:1767319304.414341:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319304.414384:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319304.414391:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985135232:0@lo:400:kworker.0 00000100:00100000:3.0:1767319304.414412:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319304.414467:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985135232 00000100:00100000:1.0:1767319304.414475:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985135232 00000100:00100000:3.0:1767319304.414476:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985134720:0@lo:38:kworker.0 00000100:00100000:3.0:1767319304.414505:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319304.414546:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134720 00000100:00100000:1.0:1767319304.414551:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134720 00000100:00100000:3.0:1767319304.414554:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4539 00000100:00100000:3.0:1767319304.414564:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985135232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319304.414573:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4987 00000100:00100000:1.0:1767319304.414580:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce18b154050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985134720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319304.414598:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319304.414609:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985135232:12345-0@lo:400:kworker.0 Request processed in 46us (194us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319304.414617:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4539 00000100:00100000:1.0:1767319304.414640:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce18b154050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985134720:12345-0@lo:38:kworker.0 Request processed in 57us (132us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319304.414648:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4987 00000100:00100000:3.0:1767319304.414660:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985135232:0@lo:400:kworker.0 00000100:00080000:3.0:1767319304.414697:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319304.414703:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319304.414706:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985134720:0@lo:38:kworker.0 00000100:00100000:3.0:1767319304.414712:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985134848:0@lo:38:kworker.0 00000100:00100000:3.0:1767319304.414741:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319304.414774:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985135104:0@lo:38:kworker.0 00000100:00100000:3.0:1767319304.414799:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319304.414824:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985135104 00000100:00100000:3.0:1767319304.414827:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985135104 00000100:00100000:3.0:1767319304.414848:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4988 00000100:00100000:3.0:1767319304.414853:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985135104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319304.414864:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319304.414895:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985135104:12345-0@lo:38:kworker.0 Request processed in 41us (96us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319304.414903:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4988 00000100:00080000:3.0:1767319304.414935:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319304.414939:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319304.414943:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985135104:0@lo:38:kworker.0 00000100:00100000:3.0:1767319304.414952:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985135360:0@lo:38:kworker.0 00000100:00100000:3.0:1767319304.414976:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319304.414997:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985135360 00000100:00100000:3.0:1767319304.415000:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985135360 00000100:00100000:3.0:1767319304.415023:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4989 00000100:00100000:3.0:1767319304.415029:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985135360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319304.415039:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319304.415070:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985135360:12345-0@lo:38:kworker.0 Request processed in 41us (95us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319304.415076:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4989 00000100:00080000:3.0:1767319304.415097:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319304.415101:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319304.415103:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985135360:0@lo:38:kworker.0 00000100:00080000:2.0:1767319304.416425:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319304.417376:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985134848 00000100:00100000:0.0:1767319304.417388:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985134848 00000100:00100000:0.0:1767319304.417448:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1737 00000100:00100000:0.0:1767319304.417457:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985134848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319304.417478:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319304.417522:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985135488 00000100:00100000:2.0:1767319304.417528:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985135488 00000100:00100000:0.0:1767319304.417564:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985134848:12345-0@lo:38:kworker.0 Request processed in 107us (2822us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319304.417572:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1737 00000100:00100000:2.0:1767319304.417586:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2986 00000100:00100000:2.0:1767319304.417596:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985135488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319304.417654:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985135488:12345-0@lo:400:kworker.0 Request processed in 61us (3312us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319304.417661:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2986 00000100:00080000:3.0:1767319304.417832:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319304.417839:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319304.417843:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985134848:0@lo:38:kworker.0 00000100:00100000:0.0:1767319304.418370:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985135488:0@lo:400:kworker.0 00000100:00080000:1.0:1767319304.418604:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319304.923516:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319304.923540:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319304.923572:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985135872:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319304.923645:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319304.923834:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985135872 00000100:00100000:2.0:1767319304.923843:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985135872 00000100:00100000:2.0:1767319304.923897:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2152 00000100:00100000:2.0:1767319304.923906:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985135872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319304.923934:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319304.923992:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985135872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (333us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319304.924000:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2152 00000100:00100000:2.0:1767319304.924048:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985135872:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319304.925865:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985135744:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319304.926792:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319304.928987:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985135744 00000100:00100000:1.0:1767319304.929002:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985135744 00000100:00100000:1.0:1767319304.929083:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2153 00000100:00100000:1.0:1767319304.929093:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985135744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319304.929124:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319304.929215:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985135744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 123us (2426us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319304.929227:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2153 00000100:00100000:0.0:1767319304.929417:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985135744:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319305.051696:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319305.388408:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319305.388929:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920553088 00000100:00100000:1.0:1767319305.388939:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920553088 00000100:00100000:1.0:1767319305.388992:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4990 00000100:00100000:1.0:1767319305.389001:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920553088:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319305.389023:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319305.389093:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920553088:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 91us (694us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319305.389101:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4990 00000100:00100000:2.0:1767319305.392795:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319305.392897:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920553216 00000100:00100000:3.0:1767319305.392906:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920553216 00000100:00100000:3.0:1767319305.392993:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4540 00000100:00100000:3.0:1767319305.393003:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920553216:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319305.393078:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920553216:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (294us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319305.393086:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4540 00000100:00080000:1.0:1767319305.393540:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319306.075519:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319307.099532:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319308.123393:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319309.148097:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319309.532143:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319309.532153:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319309.532157:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319309.532161:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319309.532186:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319309.532195:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4552 00000100:00080000:1.0:1767319309.532200:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319309.532207:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319309.532245:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7800 x1853163985136000/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319309.532271:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319309.532275:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319309.532276:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319309.532280:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4552 00000100:00080000:1.0:1767319309.532283:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319309.532287:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319309.532301:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5f80 x1853163985136128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319309.532344:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319309.532348:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319309.532350:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319309.532354:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4552 00000100:00080000:1.0:1767319309.532356:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319309.532359:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319309.532371:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6680 x1853163985136384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319309.532397:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767319309.532404:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319309.532409:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319309.532411:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319309.532415:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985136256:0@lo:400:kworker.0 00000100:00080000:1.0:1767319309.532417:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4552 00000100:00080000:1.0:1767319309.532423:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319309.532428:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319309.532442:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985136640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319309.532480:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985136512:0@lo:400:kworker.0 00000100:00100000:3.0:1767319309.532489:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319309.532557:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319309.532594:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985136896:0@lo:400:kworker.0 00000100:00100000:0.0:1767319309.532633:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985136000:0@lo:38:kworker.0 00000100:00100000:3.0:1767319309.538579:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985136768:0@lo:400:kworker.0 00000100:00100000:1.0:1767319309.538916:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136256 00000100:00100000:1.0:1767319309.538928:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136256 00000100:00100000:1.0:1767319309.538997:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4541 00000100:00100000:1.0:1767319309.539006:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985136256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319309.539087:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985136256:12345-0@lo:400:kworker.0 Request processed in 82us (6599us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319309.539094:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4541 00000100:00080000:1.0:1767319309.539115:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319309.539545:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319309.539594:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985136128:0@lo:38:kworker.0 00000100:00100000:0.0:1767319309.539625:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319309.539646:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136000 00000100:00100000:1.0:1767319309.539652:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136000 00000100:00100000:0.0:1767319309.539654:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985136384:0@lo:38:kworker.0 00000100:00100000:1.0:1767319309.539685:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4991 00000100:00100000:0.0:1767319309.539686:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319309.539695:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985136000:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319309.539712:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985136640:0@lo:38:kworker.0 00010000:00020000:1.0:1767319309.539718:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319309.539738:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319309.539766:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985136000:12345-0@lo:38:kworker.0 Request processed in 69us (907us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319309.539773:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4991 00000100:00080000:0.0:1767319309.539780:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319309.539785:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136384 00000100:00080000:0.0:1767319309.539789:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319309.539790:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136384 00000100:00100000:0.0:1767319309.539796:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985136000:0@lo:38:kworker.0 00000100:00100000:1.0:1767319309.539809:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136640 00000100:00100000:1.0:1767319309.539814:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136640 00000100:00100000:0.0:1767319309.539829:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136128 00000100:00100000:0.0:1767319309.539835:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136128 00000100:00100000:1.0:1767319309.539835:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4992 00000100:00100000:1.0:1767319309.539843:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985136384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319309.539853:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319309.539867:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1738 00000100:00100000:0.0:1767319309.539874:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985136128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319309.539886:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319309.539891:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985136384:12345-0@lo:38:kworker.0 Request processed in 44us (205us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319309.539898:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4992 00000100:00100000:1.0:1767319309.539913:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4993 00000100:00100000:1.0:1767319309.539917:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985136640:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319309.539923:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985136128:12345-0@lo:38:kworker.0 Request processed in 47us (295us total) trans 0 rc -19/-19 00010000:00020000:1.0:1767319309.539930:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319309.539934:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1738 00000100:00100000:1.0:1767319309.539963:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985136640:12345-0@lo:38:kworker.0 Request processed in 43us (225us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319309.539971:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319309.539975:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4993 00000100:00100000:1.0:1767319309.539990:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136512 00000100:00080000:0.0:1767319309.540015:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319309.540020:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319309.540023:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985136128:0@lo:38:kworker.0 00000100:00080000:0.0:1767319309.540031:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319309.540095:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136512 00000100:00100000:1.0:1767319309.540118:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4542 00000100:00100000:1.0:1767319309.540122:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985136512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319309.540147:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985136512:12345-0@lo:400:kworker.0 Request processed in 26us (181us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319309.540151:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4542 00000100:00080000:1.0:1767319309.540187:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319309.540212:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985136512:0@lo:400:kworker.0 00000100:00100000:1.0:1767319309.540242:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985136256:0@lo:400:kworker.0 00000100:00080000:0.0:1767319309.540657:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319309.540662:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985136384:0@lo:38:kworker.0 00000100:00080000:0.0:1767319309.540677:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319309.540681:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319309.540683:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985136640:0@lo:38:kworker.0 00000100:00100000:2.0:1767319309.540770:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319309.540895:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136896 00000100:00100000:0.0:1767319309.540900:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136896 00000100:00100000:0.0:1767319309.540992:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2987 00000100:00100000:0.0:1767319309.541000:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985136896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319309.541050:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985136896:12345-0@lo:400:kworker.0 Request processed in 51us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319309.541066:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2987 00000100:00100000:2.0:1767319309.541218:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985136896:0@lo:400:kworker.0 00000100:00080000:1.0:1767319309.541448:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319309.541597:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319309.541644:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985136768 00000100:00100000:2.0:1767319309.541650:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985136768 00000100:00080000:1.0:1767319309.541696:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319309.541738:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2988 00000100:00100000:2.0:1767319309.541747:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985136768:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319309.541789:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985136768:12345-0@lo:400:kworker.0 Request processed in 43us (192us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319309.541796:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2988 00000100:00100000:3.0:1767319309.541823:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985136768:0@lo:400:kworker.0 00000100:00100000:3.0:1767319310.043642:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985137024:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319310.043713:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319310.043944:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137024 00000100:00100000:2.0:1767319310.043952:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137024 00000100:00100000:2.0:1767319310.044244:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2154 00000100:00100000:2.0:1767319310.044255:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985137024:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319310.044287:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319310.044341:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985137024:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (627us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319310.044349:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2154 00000100:00100000:1.0:1767319310.044421:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985137024:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319310.044489:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985137152:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319310.044516:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319310.052527:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137152 00000100:00100000:0.0:1767319310.052544:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137152 00000100:00100000:0.0:1767319310.052632:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2155 00000100:00100000:0.0:1767319310.052643:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985137152:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319310.052678:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319310.052761:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985137152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 119us (8242us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319310.052769:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2155 00000100:00100000:3.0:1767319310.052890:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985137152:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319310.171398:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319310.507246:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319310.507470:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920553472 00000100:00100000:1.0:1767319310.507480:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920553472 00000100:00080000:2.0:1767319310.507544:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319310.507561:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4543 00000100:00100000:1.0:1767319310.507572:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920553472:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319310.507647:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920553472:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (411us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319310.507656:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4543 00000100:00100000:1.0:1767319310.507965:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319310.508094:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920553344 00000100:00100000:3.0:1767319310.508100:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920553344 00000100:00100000:3.0:1767319310.508137:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4994 00000100:00100000:3.0:1767319310.508144:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920553344:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319310.508183:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319310.508224:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920553344:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 79us (259us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319310.508231:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4994 00010000:00010000:0.0:1767319311.195559:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319312.220243:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319313.243565:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319314.268512:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319314.602290:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767319314.602480:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920553600 offset 0 length 520: 4 00000100:00100000:0.0:1767319314.605068:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920553856 00000100:00100000:0.0:1767319314.605083:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920553856 00000100:00100000:0.0:1767319314.605270:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4544 00000100:00100000:0.0:1767319314.605281:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920553856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319314.605384:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920553856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (3104us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319314.605394:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4544 00000100:00100000:3.0:1767319314.607677:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319314.607770:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920553728 00000100:00100000:0.0:1767319314.607784:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920553728 00000100:00100000:0.0:1767319314.607838:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4995 00000100:00100000:0.0:1767319314.607846:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920553728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319314.607868:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319314.607933:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920553728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 86us (269us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319314.607949:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4995 00000100:00080000:2.0:1767319314.608389:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319314.651371:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319289/real 1767319289] req@ffff8ce1a54b4e00 x1853163985130752/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319314 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319314.651445:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319314.651457:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319314.651460:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319314.651463:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319314.651469:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319314.651478:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4557 00000100:00080000:1.0:1767319314.651483:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319314.651490:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319314.651530:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4380 x1853163985137280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319314.651547:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319314.651551:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319314.651553:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319314.651557:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4557 00000100:00080000:1.0:1767319314.651559:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319314.651563:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319314.651578:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5500 x1853163985137408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319314.651664:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319314.651668:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319314.651670:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319314.651674:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4557 00000100:00080000:1.0:1767319314.651677:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319314.651680:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319314.651692:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985137664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319314.651720:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319314.651724:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319314.651727:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319314.651733:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4557 00000100:00080000:1.0:1767319314.651736:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319314.651741:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319314.651756:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5f80 x1853163985137920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319314.651771:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985137536:0@lo:400:kworker.0 00000100:00100000:1.0:1767319314.651811:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985138048:0@lo:400:kworker.0 00000100:00100000:2.0:1767319314.651905:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985138176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319314.654073:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a54b4e00 x1853163985130752/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319314 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319314.654108:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985137280:0@lo:38:kworker.0 00000100:00100000:3.0:1767319314.658945:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319314.659095:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137536 00000100:00100000:3.0:1767319314.659108:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137536 00000100:00100000:3.0:1767319314.660229:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4545 00000100:00100000:3.0:1767319314.660258:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985137536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.660662:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985137536:12345-0@lo:400:kworker.0 Request processed in 418us (1729us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319314.660670:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4545 00000100:00100000:3.0:1767319314.660743:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985137536:0@lo:400:kworker.0 00000100:00080000:3.0:1767319314.660793:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319314.660821:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985137792:0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.660856:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319314.660897:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137792 00000100:00100000:3.0:1767319314.660916:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137792 00000100:00100000:3.0:1767319314.661743:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4546 00000100:00100000:3.0:1767319314.661756:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985137792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.661814:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985137792:12345-0@lo:400:kworker.0 Request processed in 63us (957us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319314.661824:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4546 00000100:00080000:3.0:1767319314.661842:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319314.661863:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985137792:0@lo:400:kworker.0 00000100:00100000:1.0:1767319314.663485:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319314.663556:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985138048 00000100:00100000:1.0:1767319314.663564:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985138048 00000100:00100000:1.0:1767319314.663625:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2989 00000100:00100000:1.0:1767319314.663635:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985138048:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319314.663686:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985138048:12345-0@lo:400:kworker.0 Request processed in 53us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319314.663693:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2989 00000100:00080000:1.0:1767319314.663717:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319314.663748:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985138048:0@lo:400:kworker.0 00000100:00100000:2.0:1767319314.664324:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319314.664523:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985138176 00000100:00100000:1.0:1767319314.664529:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985138176 00000100:00100000:1.0:1767319314.664613:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2990 00000100:00100000:1.0:1767319314.664620:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985138176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319314.664658:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985138176:12345-0@lo:400:kworker.0 Request processed in 39us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319314.664664:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2990 00000100:00100000:0.0:1767319314.665248:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319314.665293:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985137408:0@lo:38:kworker.0 00000100:00100000:0.0:1767319314.665324:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319314.665347:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985137664:0@lo:38:kworker.0 00000100:00100000:0.0:1767319314.665365:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319314.665393:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137280 00000100:00100000:0.0:1767319314.665398:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137280 00000100:00100000:0.0:1767319314.665418:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137664 00000100:00100000:0.0:1767319314.665420:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137664 00000100:00100000:0.0:1767319314.665433:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4996 00000100:00100000:0.0:1767319314.665438:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985137280:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319314.665463:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319314.665494:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985137280:12345-0@lo:38:kworker.0 Request processed in 54us (247us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319314.665500:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4996 00000100:00100000:0.0:1767319314.665514:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4997 00000100:00100000:0.0:1767319314.665518:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985137664:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319314.665526:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319314.665549:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985137664:12345-0@lo:38:kworker.0 Request processed in 29us (182us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319314.665554:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4997 00000100:00080000:0.0:1767319314.665584:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319314.665589:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319314.665592:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985137664:0@lo:38:kworker.0 00000100:00100000:0.0:1767319314.665599:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985137920:0@lo:38:kworker.0 00000100:00100000:0.0:1767319314.665621:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319314.665641:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137920 00000100:00100000:0.0:1767319314.665644:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137920 00000100:00100000:0.0:1767319314.665659:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4998 00000100:00100000:0.0:1767319314.665663:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985137920:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319314.665672:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319314.665701:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985137920:12345-0@lo:38:kworker.0 Request processed in 36us (79us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319314.665706:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4998 00000100:00080000:0.0:1767319314.665727:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319314.665731:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319314.665734:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985137920:0@lo:38:kworker.0 00000100:00080000:0.0:1767319314.665739:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319314.665751:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319314.665755:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985130752:0@lo:250:kworker.0 00000100:00080000:0.0:1767319314.665786:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319314.665791:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319314.665794:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985137280:0@lo:38:kworker.0 00000100:00100000:1.0:1767319314.665952:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985137408 00000100:00100000:1.0:1767319314.665957:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985137408 00000100:00100000:1.0:1767319314.665993:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1739 00000100:00100000:1.0:1767319314.665999:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985137408:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319314.666010:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319314.666040:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985137408:12345-0@lo:38:kworker.0 Request processed in 40us (715us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319314.666046:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1739 00000100:00080000:1.0:1767319314.666074:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319314.666079:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319314.666082:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319314.666086:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4537 00000100:00080000:1.0:1767319314.666090:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319314.666094:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319314.666117:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7800 x1853163985138304/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319314.666329:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985138304:0@lo:250:kworker.0 00000400:00000100:0.0:1767319314.666352:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985138304 offset 0 length 520: 4 00000100:00080000:0.0:1767319314.666378:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319314.666382:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319314.666385:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985137408:0@lo:38:kworker.0 00000100:00100000:2.0:1767319314.666729:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985138176:0@lo:400:kworker.0 00000100:00080000:3.0:1767319314.666814:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319315.166237:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319315.166259:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985138432:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319315.166328:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319315.166889:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985138432 00000100:00100000:1.0:1767319315.166898:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985138432 00000100:00100000:1.0:1767319315.166980:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2156 00000100:00100000:1.0:1767319315.166989:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985138432:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319315.167032:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319315.167091:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985138432:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (762us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319315.167099:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2156 00000100:00100000:0.0:1767319315.167311:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985138432:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319315.169142:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319315.169179:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985138560:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319315.169243:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319315.169272:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985138560 00000100:00100000:2.0:1767319315.169280:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985138560 00000100:00100000:2.0:1767319315.169325:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2157 00000100:00100000:2.0:1767319315.169334:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985138560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319315.169369:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319315.169416:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985138560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (172us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319315.169424:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2157 00000100:00100000:0.0:1767319315.169471:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985138560:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319315.291569:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319316.321574:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319317.339439:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319318.365134:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319319.388582:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319319.726574:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319319.727453:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920553984 00000100:00100000:0.0:1767319319.727970:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319319.728252:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920554112 00000100:00100000:1.0:1767319319.728264:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920554112 00000100:00100000:1.0:1767319319.728361:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4547 00000100:00100000:1.0:1767319319.728374:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920554112:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319319.728498:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920554112:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 125us (533us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319319.728515:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4547 00000100:00100000:3.0:1767319319.728521:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920553984 00000100:00100000:3.0:1767319319.728601:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4999 00000100:00100000:3.0:1767319319.728609:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920553984:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319319.728644:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319319.728716:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920553984:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 105us (2172us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319319.728725:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4999 00000100:00080000:0.0:1767319319.729563:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319319.774322:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319319.774339:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319319.774345:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319319.774350:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319319.774356:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319319.774369:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4562 00000100:00080000:0.0:1767319319.774376:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319319.774385:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319319.774443:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60700 x1853163985138688/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319319.774472:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319319.774476:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319319.774477:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319319.774482:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4562 00000100:00080000:0.0:1767319319.774485:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319319.774489:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319319.774503:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60380 x1853163985138816/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319319.774550:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319319.774555:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319319.774556:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319319.774560:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4562 00000100:00080000:0.0:1767319319.774563:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319319.774566:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319319.774577:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60e00 x1853163985139072/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319319.774636:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319319.774663:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985138944:0@lo:400:kworker.0 00000100:00080000:0.0:1767319319.774666:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319319.774674:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319319.774676:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319319.774681:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4562 00000100:00080000:0.0:1767319319.774685:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319319.774690:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319319.774707:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63b80 x1853163985139328/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319319.774738:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319319.774763:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985139584:0@lo:400:kworker.0 00000100:00100000:0.0:1767319319.774764:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985139456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319319.774774:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319319.774861:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985139200:0@lo:400:kworker.0 00000100:00100000:3.0:1767319319.785777:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985138688:0@lo:38:kworker.0 00000100:00100000:3.0:1767319319.785877:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319319.785914:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985138816:0@lo:38:kworker.0 00000100:00100000:3.0:1767319319.785952:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319319.785981:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985139072:0@lo:38:kworker.0 00000100:00100000:3.0:1767319319.786011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319319.786044:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985139328:0@lo:38:kworker.0 00000100:00100000:3.0:1767319319.786067:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319319.786115:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985138944 00000100:00100000:3.0:1767319319.786125:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985138944 00000100:00100000:3.0:1767319319.786354:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985138688 00000100:00100000:3.0:1767319319.786370:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985138688 00000100:00100000:3.0:1767319319.786502:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139072 00000100:00100000:3.0:1767319319.786508:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139072 00000100:00100000:3.0:1767319319.786522:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139328 00000100:00100000:3.0:1767319319.786524:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139328 00000100:00100000:3.0:1767319319.786541:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4548 00000100:00100000:3.0:1767319319.786557:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985138944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319319.786649:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319319.786675:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985138944:12345-0@lo:400:kworker.0 Request processed in 121us (11914us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319319.786684:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4548 00000100:00100000:3.0:1767319319.786719:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5000 00000100:00100000:0.0:1767319319.786722:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985138816 00000100:00100000:3.0:1767319319.786731:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985138688:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319319.786733:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985138816 00010000:00020000:3.0:1767319319.786770:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319319.786777:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1740 00000100:00100000:0.0:1767319319.786787:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985138816:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319319.786807:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319319.786827:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985138688:12345-0@lo:38:kworker.0 Request processed in 97us (958us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319319.786835:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5000 00000100:00100000:0.0:1767319319.786860:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985138816:12345-0@lo:38:kworker.0 Request processed in 69us (904us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319319.786879:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5001 00000100:00100000:0.0:1767319319.786886:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1740 00000100:00100000:3.0:1767319319.786894:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985139072:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319319.786908:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319319.786919:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319319.786945:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5002 00000100:00100000:0.0:1767319319.786958:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985139328:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319319.786959:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985139072:12345-0@lo:38:kworker.0 Request processed in 58us (938us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319319.786975:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5001 00010000:00020000:0.0:1767319319.786980:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319319.787028:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985139328:12345-0@lo:38:kworker.0 Request processed in 73us (958us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319319.787040:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5002 00000100:00080000:3.0:1767319319.787046:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319319.787055:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319319.787061:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985138688:0@lo:38:kworker.0 00000100:00100000:0.0:1767319319.787080:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139456 00000100:00080000:3.0:1767319319.787085:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319319.787087:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139456 00000100:00080000:3.0:1767319319.787094:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319319.787099:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985138816:0@lo:38:kworker.0 00000100:00080000:3.0:1767319319.787113:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319319.787120:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319319.787124:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985139072:0@lo:38:kworker.0 00000100:00080000:3.0:1767319319.787139:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319319.787143:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2991 00000100:00080000:3.0:1767319319.787150:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319319.787190:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985139456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319319.787207:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985139328:0@lo:38:kworker.0 00000100:00100000:1.0:1767319319.787236:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319319.787260:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985139456:12345-0@lo:400:kworker.0 Request processed in 107us (614us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319319.787268:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2991 00000100:00100000:3.0:1767319319.787293:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985138944:0@lo:400:kworker.0 00000100:00100000:1.0:1767319319.787308:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139584 00000100:00100000:1.0:1767319319.787315:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139584 00000100:00100000:1.0:1767319319.787354:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2992 00000100:00100000:1.0:1767319319.787364:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985139584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319319.787406:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985139584:12345-0@lo:400:kworker.0 Request processed in 44us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319319.787412:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2992 00000100:00100000:1.0:1767319319.787456:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985139584:0@lo:400:kworker.0 00000100:00080000:1.0:1767319319.787489:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319319.787492:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319319.787512:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985139456:0@lo:400:kworker.0 00000100:00100000:2.0:1767319319.792842:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319319.792968:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139200 00000100:00100000:3.0:1767319319.792979:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139200 00000100:00100000:3.0:1767319319.793094:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4549 00000100:00100000:3.0:1767319319.793104:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985139200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319319.793162:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985139200:12345-0@lo:400:kworker.0 Request processed in 59us (328us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319319.793197:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4549 00000100:00080000:1.0:1767319319.793264:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319319.793674:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985139200:0@lo:400:kworker.0 00000100:00100000:0.0:1767319320.283512:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985139712:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319320.283586:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319320.283687:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139712 00000100:00100000:1.0:1767319320.283695:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139712 00000100:00100000:1.0:1767319320.283748:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2158 00000100:00100000:1.0:1767319320.283757:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985139712:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319320.283785:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319320.283833:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985139712:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (246us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319320.283840:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2158 00000100:00100000:1.0:1767319320.283894:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985139712:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319320.285427:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319320.285448:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985139840:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319320.285507:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319320.289571:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139840 00000100:00100000:2.0:1767319320.289588:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139840 00000100:00100000:2.0:1767319320.289660:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2159 00000100:00100000:2.0:1767319320.289670:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985139840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319320.289703:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319320.289795:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985139840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 127us (4287us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319320.289804:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2159 00000100:00100000:0.0:1767319320.289938:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985139840:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319320.419763:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319321.436232:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319322.460447:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319323.484890:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319324.507628:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319324.844544:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319324.846072:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920554240 00000100:00100000:3.0:1767319324.846087:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920554240 00000100:00100000:3.0:1767319324.846288:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5003 00000100:00100000:3.0:1767319324.846296:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920554240:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319324.846320:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319324.846391:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920554240:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (1852us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319324.846402:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5003 00000100:00100000:1.0:1767319324.846540:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319324.846664:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920554368 00000100:00100000:2.0:1767319324.846669:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920554368 00000100:00100000:2.0:1767319324.846741:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4550 00000100:00100000:2.0:1767319324.846750:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920554368:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319324.846810:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920554368:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 62us (272us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319324.846818:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4550 00000100:00080000:1.0:1767319324.846941:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319324.891417:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319324.891427:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319324.891430:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319324.891433:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319324.891438:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319324.891445:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4568 00000100:00080000:1.0:1767319324.891450:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319324.891456:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319324.891510:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985139968/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319324.891527:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319324.891530:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319324.891532:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319324.891535:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4568 00000100:00080000:1.0:1767319324.891537:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319324.891540:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319324.891552:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4000 x1853163985140096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319324.891583:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319324.891586:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319324.891588:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319324.891591:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4568 00000100:00080000:1.0:1767319324.891593:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319324.891596:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319324.891604:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4e00 x1853163985140352/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319324.891630:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319324.891633:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319324.891634:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319324.891637:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4568 00000100:00080000:1.0:1767319324.891640:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319324.891642:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319324.891650:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4380 x1853163985140608/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319324.891689:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985140480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.891815:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985140224:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.891868:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319324.891895:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.891931:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319324.891931:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319324.891942:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985140736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.891945:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985140864:0@lo:400:kworker.0 00000100:00100000:1.0:1767319324.891974:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.891993:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319324.892019:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140224 00000100:00100000:1.0:1767319324.892026:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140224 00000100:00100000:0.0:1767319324.892032:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140736 00000100:00100000:0.0:1767319324.892037:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140736 00000100:00100000:1.0:1767319324.892094:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140480 00000100:00100000:0.0:1767319324.892096:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140864 00000100:00100000:1.0:1767319324.892099:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140480 00000100:00100000:0.0:1767319324.892110:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140864 00000100:00100000:1.0:1767319324.892131:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4551 00000100:00100000:1.0:1767319324.892142:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985140224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.892146:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2993 00000100:00100000:0.0:1767319324.892156:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985140736:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.892218:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985140736:12345-0@lo:400:kworker.0 Request processed in 64us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319324.892225:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2993 00000100:00100000:0.0:1767319324.892257:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2994 00000100:00100000:0.0:1767319324.892263:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985140864:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.892297:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985140864:12345-0@lo:400:kworker.0 Request processed in 34us (305us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319324.892303:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2994 00000100:00100000:2.0:1767319324.892353:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4552 00000100:00100000:2.0:1767319324.892364:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985140480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319324.892400:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985140480:12345-0@lo:400:kworker.0 Request processed in 39us (505us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319324.892406:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4552 00000100:00100000:1.0:1767319324.892549:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985140224:12345-0@lo:400:kworker.0 Request processed in 410us (680us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319324.892555:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4551 00000100:00080000:1.0:1767319324.892571:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319324.892574:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319324.892576:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319324.893546:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985140736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.893561:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985140480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.893597:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985140864:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.893606:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985140224:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.893643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985139968:0@lo:38:kworker.0 00000100:00100000:0.0:1767319324.893684:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.893738:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985140096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319324.893762:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.893797:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985140352:0@lo:38:kworker.0 00000100:00100000:0.0:1767319324.893831:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.893849:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985140608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319324.893869:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319324.894142:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985139968 00000100:00100000:2.0:1767319324.894150:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985139968 00000100:00100000:3.0:1767319324.894193:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140352 00000100:00100000:3.0:1767319324.894207:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140352 00000100:00100000:2.0:1767319324.894219:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140608 00000100:00100000:2.0:1767319324.894226:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140608 00000100:00100000:3.0:1767319324.894265:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5004 00000100:00100000:3.0:1767319324.894275:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985139968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319324.894301:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319324.894377:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985139968:12345-0@lo:38:kworker.0 Request processed in 102us (693us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319324.894385:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5004 00000100:00100000:3.0:1767319324.894404:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5005 00000100:00100000:3.0:1767319324.894408:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985140352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319324.894421:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319324.894449:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985140352:12345-0@lo:38:kworker.0 Request processed in 36us (615us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319324.894457:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5005 00000100:00100000:3.0:1767319324.894476:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5006 00000100:00100000:3.0:1767319324.894480:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985140608:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319324.894488:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319324.894512:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985140608:12345-0@lo:38:kworker.0 Request processed in 31us (642us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319324.894517:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5006 00000100:00080000:1.0:1767319324.900145:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319324.900159:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319324.900180:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985139968:0@lo:38:kworker.0 00000100:00080000:1.0:1767319324.900208:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319324.900212:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319324.900214:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985140352:0@lo:38:kworker.0 00000100:00080000:1.0:1767319324.900226:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319324.900275:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319324.900278:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985140608:0@lo:38:kworker.0 00000100:00100000:1.0:1767319324.900325:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140096 00000100:00100000:1.0:1767319324.900332:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140096 00000100:00100000:1.0:1767319324.900391:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1741 00000100:00100000:1.0:1767319324.900399:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985140096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319324.900421:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319324.900501:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985140096:12345-0@lo:38:kworker.0 Request processed in 100us (6736us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319324.900509:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1741 00000100:00080000:0.0:1767319324.900807:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319324.900813:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319324.900817:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985140096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319325.403395:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319325.403408:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319325.403428:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985141120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319325.403428:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985140992:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319325.403623:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319325.405103:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985141120 00000100:00100000:0.0:1767319325.405115:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985141120 00000100:00100000:0.0:1767319325.405206:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2160 00000100:00100000:0.0:1767319325.405216:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985141120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319325.405247:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319325.405321:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985141120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 107us (1698us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319325.405329:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2160 00000100:00100000:1.0:1767319325.405446:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985141120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319325.405989:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319325.406045:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985140992 00000100:00100000:3.0:1767319325.406051:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985140992 00000100:00100000:3.0:1767319325.406087:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2161 00000100:00100000:3.0:1767319325.406095:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985140992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319325.406114:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319325.406190:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985140992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (174us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319325.406197:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2161 00000100:00100000:2.0:1767319325.406339:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985140992:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319325.533444:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319326.556421:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319327.581438:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319328.605489:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319329.627720:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319329.962929:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319329.963225:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920554624 00000100:00100000:2.0:1767319329.963236:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920554624 00000100:00100000:2.0:1767319329.963351:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4553 00000100:00100000:2.0:1767319329.963363:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920554624:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319329.963469:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920554624:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 109us (549us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319329.963477:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4553 00000100:00080000:1.0:1767319329.963652:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319329.969812:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319329.970139:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920554496 00000100:00100000:0.0:1767319329.970149:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920554496 00000100:00100000:0.0:1767319329.971114:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5007 00000100:00100000:0.0:1767319329.971124:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920554496:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319329.971204:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319329.971301:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920554496:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 176us (1499us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319329.971310:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5007 00000100:00080000:1.0:1767319330.011799:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319330.011808:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319330.011811:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319330.011816:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319330.011825:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319330.011833:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4573 00000100:00080000:1.0:1767319330.011838:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319330.011845:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319330.011885:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5180 x1853163985141248/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319330.011956:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985141248:0@lo:38:kworker.0 00000100:00100000:2.0:1767319330.012017:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319330.012097:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319330.012101:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319330.012103:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319330.012108:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4573 00000100:00080000:1.0:1767319330.012110:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319330.012114:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319330.012129:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985141376/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319330.012754:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985141248 00000100:00100000:3.0:1767319330.012763:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985141248 00000100:00100000:3.0:1767319330.014149:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5008 00000100:00100000:3.0:1767319330.014160:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985141248:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319330.014193:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319330.015058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985141376:0@lo:38:kworker.0 00000100:00100000:2.0:1767319330.015145:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319330.015274:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985141376 00000100:00100000:1.0:1767319330.015282:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985141376 00000100:00100000:1.0:1767319330.015312:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1742 00000100:00100000:1.0:1767319330.015318:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985141376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319330.015334:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319330.015516:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985141376:12345-0@lo:38:kworker.0 Request processed in 196us (369us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319330.015608:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1742 00000100:00080000:1.0:1767319330.015668:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319330.015672:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319330.015687:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319330.015693:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4573 00000100:00080000:1.0:1767319330.015696:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319330.015701:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319330.015712:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985141504:0@lo:400:kworker.0 00000100:00100000:1.0:1767319330.015761:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319330.022741:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319330.022745:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985141248:12345-0@lo:38:kworker.0 Request processed in 8576us (10718us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319330.022752:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319330.022759:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985141248:0@lo:38:kworker.0 00000100:00100000:3.0:1767319330.022766:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5008 00000100:00080000:2.0:1767319330.022795:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319330.022799:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319330.022802:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985141376:0@lo:38:kworker.0 00000100:00100000:1.0:1767319330.023816:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5f80 x1853163985141632/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319330.023907:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319330.023911:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319330.023914:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319330.023919:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4573 00000100:00080000:1.0:1767319330.023922:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319330.023928:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319330.023943:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985141760:0@lo:400:kworker.0 00000100:00100000:1.0:1767319330.024003:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319330.024063:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985141504 00000100:00100000:3.0:1767319330.024076:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985141504 00000100:00100000:3.0:1767319330.024137:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4554 00000100:00100000:3.0:1767319330.024147:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985141504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319330.024223:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985141504:12345-0@lo:400:kworker.0 Request processed in 78us (8460us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319330.024236:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4554 00000100:00100000:3.0:1767319330.024397:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985141632:0@lo:38:kworker.0 00000100:00080000:3.0:1767319330.024441:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319330.027199:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0a80 x1853163985141888/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319330.027249:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985142016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319330.027437:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319330.027926:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985142144:0@lo:400:kworker.0 00000100:00100000:3.0:1767319330.027955:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319330.027995:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142016 00000100:00100000:3.0:1767319330.028003:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142016 00000100:00100000:3.0:1767319330.028052:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142144 00000100:00100000:3.0:1767319330.028055:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142144 00000100:00100000:3.0:1767319330.028077:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2995 00000100:00100000:3.0:1767319330.028085:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985142016:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319330.028134:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985142016:12345-0@lo:400:kworker.0 Request processed in 49us (698us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319330.028141:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2995 00000100:00100000:3.0:1767319330.028157:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2996 00000100:00100000:3.0:1767319330.028186:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985142144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319330.028220:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985142144:12345-0@lo:400:kworker.0 Request processed in 57us (264us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319330.028225:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2996 00000100:00100000:3.0:1767319330.028921:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985142016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319330.028962:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985142144:0@lo:400:kworker.0 00000100:00080000:3.0:1767319330.028977:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319330.028981:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319330.033325:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985141504:0@lo:400:kworker.0 00000100:00100000:1.0:1767319330.033367:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319330.033407:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985141760 00000100:00100000:3.0:1767319330.033415:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985141760 00000100:00100000:3.0:1767319330.033477:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985141632 00000100:00100000:3.0:1767319330.033480:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985141632 00000100:00100000:3.0:1767319330.033503:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4555 00000100:00100000:3.0:1767319330.033513:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985141760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319330.033588:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985141760:12345-0@lo:400:kworker.0 Request processed in 76us (9583us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319330.033597:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5009 00000100:00100000:1.0:1767319330.033632:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985141888:0@lo:38:kworker.0 00000100:00100000:1.0:1767319330.033665:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319330.033724:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985141760:0@lo:400:kworker.0 00000100:00100000:1.0:1767319330.033741:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985141888 00000100:00100000:1.0:1767319330.033745:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985141888 00000100:00100000:2.0:1767319330.033771:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985141632:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319330.033791:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319330.033828:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985141632:12345-0@lo:38:kworker.0 Request processed in 223us (462us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319330.035905:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4555 00000100:00100000:1.0:1767319330.035943:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5010 00000100:00100000:1.0:1767319330.035953:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985141888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319330.035975:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319330.036044:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985141888:12345-0@lo:38:kworker.0 Request processed in 91us (2377us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319330.036050:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5010 00000100:00080000:1.0:1767319330.036099:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319330.036104:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319330.036108:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985141888:0@lo:38:kworker.0 00000100:00080000:1.0:1767319330.036121:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319330.036124:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319330.036127:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985141632:0@lo:38:kworker.0 00000100:00080000:1.0:1767319330.036248:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319330.036723:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5009 00000100:00100000:1.0:1767319330.524052:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319330.524055:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319330.524079:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985142400:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319330.524080:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985142272:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319330.524409:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319330.527709:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142400 00000100:00100000:0.0:1767319330.527725:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142400 00000100:00100000:0.0:1767319330.527797:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2162 00000100:00100000:0.0:1767319330.527809:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985142400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319330.529639:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319330.529851:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985142400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2040us (5438us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319330.529863:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2162 00000100:00100000:1.0:1767319330.529997:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319330.530476:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142272 00000100:00100000:0.0:1767319330.530485:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142272 00000100:00100000:0.0:1767319330.530535:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2163 00000100:00100000:0.0:1767319330.530543:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985142272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319330.530572:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319330.530621:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985142272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (625us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319330.530628:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2163 00000100:00100000:1.0:1767319330.530805:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985142272:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319330.536493:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985142400:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319330.651651:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319331.676407:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319332.699462:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319333.725337:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319334.747584:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319335.082923:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319335.082972:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319335.083065:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920554752 00000100:00100000:0.0:1767319335.083075:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920554752 00000100:00100000:0.0:1767319335.083129:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920554880 00000100:00100000:0.0:1767319335.083132:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920554880 00000100:00100000:0.0:1767319335.083221:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4556 00000100:00100000:0.0:1767319335.083279:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920554880:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319335.083375:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920554880:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 145us (401us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319335.083384:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4556 00000100:00100000:0.0:1767319335.083403:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5011 00000100:00100000:0.0:1767319335.083407:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920554752:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319335.083430:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319335.083456:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920554752:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (544us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319335.083462:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5011 00000100:00080000:1.0:1767319335.083907:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319335.132731:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319335.132741:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319335.132744:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319335.132747:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319335.132751:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319335.132759:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4578 00000100:00080000:2.0:1767319335.132764:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319335.132770:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319335.132808:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4a80 x1853163985142528/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319335.132825:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319335.132828:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319335.132830:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319335.132833:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4578 00000100:00080000:2.0:1767319335.132836:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319335.132839:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319335.132853:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5180 x1853163985142656/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319335.132892:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319335.132896:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319335.132898:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319335.132942:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4578 00000100:00080000:2.0:1767319335.132944:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319335.132947:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319335.132959:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7b80 x1853163985142912/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319335.132988:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319335.132992:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319335.132993:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319335.132996:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4578 00000100:00080000:2.0:1767319335.132999:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319335.133002:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319335.133014:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06d80 x1853163985143168/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319335.133057:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985143296:0@lo:400:kworker.0 00000100:00100000:2.0:1767319335.133121:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319335.133200:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319335.133207:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985143040:0@lo:400:kworker.0 00000100:00100000:2.0:1767319335.133238:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319335.133278:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985143296 00000100:00100000:2.0:1767319335.133285:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985143296 00000100:00100000:2.0:1767319335.133338:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2997 00000100:00100000:2.0:1767319335.133347:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985143296:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319335.133399:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985143296:12345-0@lo:400:kworker.0 Request processed in 54us (278us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319335.133406:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2997 00000100:00100000:2.0:1767319335.133433:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985143040 00000100:00100000:2.0:1767319335.133437:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985143040 00000100:00100000:2.0:1767319335.133465:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4557 00000100:00100000:2.0:1767319335.133471:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985143040:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319335.133500:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985143040:12345-0@lo:400:kworker.0 Request processed in 29us (262us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319335.133506:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4557 00000100:00080000:2.0:1767319335.133525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319335.133529:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319335.133562:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985143040:0@lo:400:kworker.0 00000100:00100000:2.0:1767319335.133571:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985143296:0@lo:400:kworker.0 00000100:00100000:3.0:1767319335.133669:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985142784:0@lo:400:kworker.0 00000100:00100000:3.0:1767319335.133697:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319335.133721:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319335.133726:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985143424:0@lo:400:kworker.0 00000100:00100000:3.0:1767319335.133746:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319335.133771:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985143424 00000100:00100000:3.0:1767319335.133775:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985143424 00000100:00100000:3.0:1767319335.133803:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2998 00000100:00100000:3.0:1767319335.133809:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985143424:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319335.133841:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985143424:12345-0@lo:400:kworker.0 Request processed in 32us (94us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319335.133847:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2998 00000100:00080000:3.0:1767319335.133884:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319335.133903:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985143424:0@lo:400:kworker.0 00000100:00100000:2.0:1767319335.136274:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142784 00000100:00100000:2.0:1767319335.136287:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142784 00000100:00100000:2.0:1767319335.136356:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4558 00000100:00100000:2.0:1767319335.136364:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985142784:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319335.136439:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985142784:12345-0@lo:400:kworker.0 Request processed in 75us (2740us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319335.136446:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4558 00000100:00080000:3.0:1767319335.136552:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319335.136597:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985142784:0@lo:400:kworker.0 00000100:00100000:1.0:1767319335.136782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985142528:0@lo:38:kworker.0 00000100:00100000:1.0:1767319335.136823:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319335.136863:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985142656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319335.136886:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319335.136918:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985142912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319335.136941:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319335.136966:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985143168:0@lo:38:kworker.0 00000100:00100000:1.0:1767319335.136987:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319335.137023:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142656 00000100:00100000:3.0:1767319335.137028:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142656 00000100:00100000:1.0:1767319335.137041:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142528 00000100:00100000:1.0:1767319335.137046:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142528 00000100:00100000:3.0:1767319335.137054:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1743 00000100:00100000:3.0:1767319335.137060:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985142656:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319335.137070:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985142912 00000100:00100000:1.0:1767319335.137074:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985142912 00010000:00020000:3.0:1767319335.137080:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319335.137087:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985143168 00000100:00100000:1.0:1767319335.137090:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985143168 00000100:00100000:1.0:1767319335.137110:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5012 00000100:00100000:1.0:1767319335.137116:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985142528:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319335.137123:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985142656:12345-0@lo:38:kworker.0 Request processed in 60us (234us total) trans 0 rc -19/-19 00010000:00020000:1.0:1767319335.137131:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319335.137133:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1743 00000100:00100000:3.0:1767319335.137191:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5013 00000100:00100000:3.0:1767319335.137197:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985142912:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319335.137204:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985142528:12345-0@lo:38:kworker.0 Request processed in 48us (341us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767319335.137210:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319335.137220:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5012 00000100:00100000:1.0:1767319335.137234:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5014 00000100:00100000:1.0:1767319335.137239:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985143168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319335.137258:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319335.137280:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985142912:12345-0@lo:38:kworker.0 Request processed in 81us (336us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319335.137288:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5013 00000100:00100000:1.0:1767319335.137294:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985143168:12345-0@lo:38:kworker.0 Request processed in 52us (305us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319335.137302:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5014 00000100:00080000:1.0:1767319335.137343:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319335.137348:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319335.137352:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985142528:0@lo:38:kworker.0 00000100:00080000:1.0:1767319335.137362:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319335.137366:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319335.137369:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985142656:0@lo:38:kworker.0 00000100:00080000:1.0:1767319335.137377:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319335.137380:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319335.137382:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985142912:0@lo:38:kworker.0 00000100:00080000:1.0:1767319335.137391:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319335.137394:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319335.137396:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985143168:0@lo:38:kworker.0 00000100:00100000:3.0:1767319335.643434:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985143552:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319335.643505:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319335.643543:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985143552 00000100:00100000:1.0:1767319335.643551:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985143552 00000100:00100000:1.0:1767319335.643617:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2164 00000100:00100000:1.0:1767319335.643627:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985143552:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319335.643670:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319335.643724:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985143552:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (219us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319335.643732:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2164 00000100:00100000:3.0:1767319335.643769:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985143552:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319335.643803:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985143680:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319335.643835:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319335.646374:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985143680 00000100:00100000:1.0:1767319335.646388:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985143680 00000100:00100000:1.0:1767319335.646458:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2165 00000100:00100000:1.0:1767319335.646467:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985143680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319335.646498:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319335.646601:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985143680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 135us (2765us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319335.646610:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2165 00000100:00100000:2.0:1767319335.646726:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985143680:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319335.771433:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319336.795433:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319337.820244:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319338.843372:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319339.230062:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319314/real 1767319314] req@ffff8ce1a54b7800 x1853163985138304/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319339 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319339.230108:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a54b7800 x1853163985138304/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319339 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319339.230128:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319339.230253:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319339.230258:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985138304:0@lo:250:kworker.0 00000100:00080000:3.0:1767319339.233736:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319339.233751:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319339.233756:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319339.233767:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4562 00000100:00080000:3.0:1767319339.233773:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319339.233781:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319339.233835:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1c00 x1853163985143808/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319339.233877:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319339.233882:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319339.233884:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319339.233888:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4583 00000100:00080000:3.0:1767319339.233891:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319339.233894:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319339.233921:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1180 x1853163985143936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319339.233936:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319339.233939:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319339.233940:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319339.233944:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4583 00000100:00080000:3.0:1767319339.233946:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319339.233950:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319339.233963:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1f80 x1853163985144064/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319339.234015:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319339.234019:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319339.234021:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319339.234026:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4583 00000100:00080000:3.0:1767319339.234028:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319339.234032:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319339.234052:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2a00 x1853163985144320/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319339.234086:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319339.234090:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319339.234091:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319339.234095:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4583 00000100:00080000:3.0:1767319339.234098:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319339.234101:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319339.234112:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985144576/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319339.234199:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985144704:0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.234286:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319339.234983:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319339.234995:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985144448:0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.235034:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319339.235085:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144704 00000100:00100000:3.0:1767319339.235094:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144704 00000100:00100000:2.0:1767319339.235455:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985143808:0@lo:250:kworker.0 00000400:00000100:2.0:1767319339.235486:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985143808 offset 0 length 520: 4 00000100:00100000:2.0:1767319339.235517:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985143936:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.235540:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.235561:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985144064:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.235586:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.235608:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985144320:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.235629:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.235646:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985144576:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.235667:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.235706:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144448 00000100:00100000:2.0:1767319339.235711:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144448 00000100:00100000:2.0:1767319339.235745:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985143936 00000100:00100000:2.0:1767319339.235749:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985143936 00000100:00100000:2.0:1767319339.235764:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144320 00000100:00100000:2.0:1767319339.235766:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144320 00000100:00100000:2.0:1767319339.235775:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144576 00000100:00100000:2.0:1767319339.235777:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144576 00000100:00100000:2.0:1767319339.235789:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4559 00000100:00100000:2.0:1767319339.235799:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985144448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319339.235878:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985144448:12345-0@lo:400:kworker.0 Request processed in 66us (828us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319339.235886:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4559 00000100:00100000:2.0:1767319339.235981:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5015 00000100:00100000:2.0:1767319339.235987:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985143936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319339.236008:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319339.236047:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985143936:12345-0@lo:38:kworker.0 Request processed in 59us (505us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319339.236053:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5015 00000100:00100000:2.0:1767319339.236066:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5016 00000100:00100000:2.0:1767319339.236070:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985144320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319339.236079:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319339.236099:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985144320:12345-0@lo:38:kworker.0 Request processed in 28us (469us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319339.236104:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5016 00000100:00100000:2.0:1767319339.236113:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5017 00000100:00100000:2.0:1767319339.236117:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985144576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319339.236125:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319339.236142:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985144576:12345-0@lo:38:kworker.0 Request processed in 24us (474us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319339.236147:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5017 00000100:00100000:3.0:1767319339.236701:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2999 00000100:00100000:3.0:1767319339.236717:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985144704:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.236764:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985144704:12345-0@lo:400:kworker.0 Request processed in 55us (2478us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319339.236771:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2999 00000100:00080000:3.0:1767319339.236796:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319339.236800:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319339.236914:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985144192:0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.236955:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319339.236984:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319339.236990:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985144832:0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.237015:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319339.237061:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144192 00000100:00100000:0.0:1767319339.237067:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144192 00000100:00100000:0.0:1767319339.237106:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4560 00000100:00100000:0.0:1767319339.237113:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985144192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.237149:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985144192:12345-0@lo:400:kworker.0 Request processed in 36us (192us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319339.237155:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4560 00000100:00080000:0.0:1767319339.237213:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319339.237250:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985144192:0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.238753:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144832 00000100:00100000:3.0:1767319339.238767:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144832 00000100:00100000:3.0:1767319339.238843:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3000 00000100:00100000:3.0:1767319339.238853:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985144832:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.238951:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985144832:12345-0@lo:400:kworker.0 Request processed in 99us (1934us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319339.238960:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3000 00000100:00100000:3.0:1767319339.239014:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985144448:0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.239027:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985144704:0@lo:400:kworker.0 00000100:00080000:3.0:1767319339.239083:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319339.239090:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319339.239094:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985143936:0@lo:38:kworker.0 00000100:00080000:3.0:1767319339.239106:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319339.239109:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319339.239112:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985144320:0@lo:38:kworker.0 00000100:00080000:3.0:1767319339.239122:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319339.239126:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319339.239130:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985144576:0@lo:38:kworker.0 00000100:00080000:0.0:1767319339.242260:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319339.242607:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985144832:0@lo:400:kworker.0 00000100:00100000:2.0:1767319339.242782:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144064 00000100:00100000:2.0:1767319339.242794:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144064 00000100:00100000:2.0:1767319339.243060:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1744 00000100:00100000:2.0:1767319339.243071:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985144064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319339.243103:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319339.243242:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985144064:12345-0@lo:38:kworker.0 Request processed in 171us (7653us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319339.243252:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1744 00000100:00080000:0.0:1767319339.243346:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319339.243353:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319339.243357:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985144064:0@lo:38:kworker.0 00010000:00010000:0.0:1767319339.869327:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319340.205624:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319340.205674:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767319340.205702:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920555264 offset 0 length 520: 4 00000100:00100000:1.0:1767319340.205821:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920555136 00000100:00100000:1.0:1767319340.205835:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920555136 00000100:00100000:1.0:1767319340.205930:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920555008 00000100:00100000:1.0:1767319340.205934:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920555008 00000100:00100000:1.0:1767319340.205955:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4561 00000100:00100000:1.0:1767319340.205965:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920555136:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319340.206048:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920555136:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (440us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319340.206054:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4561 00000100:00100000:1.0:1767319340.206069:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5018 00000100:00100000:1.0:1767319340.206072:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920555008:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319340.206093:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319340.206110:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920555008:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 37us (437us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319340.206114:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5018 00000100:00080000:0.0:1767319340.206722:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319340.764010:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319340.764046:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985144960:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319340.764137:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319340.764259:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985144960 00000100:00100000:1.0:1767319340.764269:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985144960 00000100:00100000:1.0:1767319340.764331:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2166 00000100:00100000:1.0:1767319340.764343:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985144960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319340.764380:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319340.764441:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985144960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319340.764449:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2166 00000100:00100000:2.0:1767319340.764652:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985144960:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319340.765595:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319340.765616:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985145088:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319340.765679:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319340.765755:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145088 00000100:00100000:0.0:1767319340.765763:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145088 00000100:00100000:0.0:1767319340.765834:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2167 00000100:00100000:0.0:1767319340.765844:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985145088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319340.765892:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319340.765950:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985145088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 107us (270us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319340.765958:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2167 00000100:00100000:2.0:1767319340.766099:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985145088:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319340.891468:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319341.915707:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319342.946604:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319343.964793:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319344.347700:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319344.347709:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319344.347712:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319344.347717:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319344.347721:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319344.347729:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4587 00000100:00080000:3.0:1767319344.347734:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319344.347741:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319344.347781:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60e00 x1853163985145216/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319344.347818:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319344.347822:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319344.347823:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319344.347827:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4587 00000100:00080000:3.0:1767319344.347830:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319344.347833:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319344.347847:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60380 x1853163985145344/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319344.347956:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319344.347960:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319344.347963:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319344.347966:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4587 00000100:00080000:3.0:1767319344.347969:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319344.347972:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319344.347989:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0380 x1853163985145600/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319344.348013:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319344.348017:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319344.348019:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319344.348022:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4587 00000100:00080000:3.0:1767319344.348025:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319344.348027:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319344.348038:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3b80 x1853163985145856/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319344.348087:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985145472:0@lo:400:kworker.0 00000100:00100000:3.0:1767319344.348158:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319344.348234:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319344.348241:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985146112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319344.348269:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319344.348306:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145472 00000100:00100000:3.0:1767319344.348313:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145472 00000100:00100000:3.0:1767319344.348366:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4562 00000100:00100000:3.0:1767319344.348376:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985145472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319344.348437:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985145472:12345-0@lo:400:kworker.0 Request processed in 62us (278us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319344.348445:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4562 00000100:00100000:3.0:1767319344.348469:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985146112 00000100:00100000:3.0:1767319344.348472:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985146112 00000100:00100000:3.0:1767319344.348499:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3001 00000100:00100000:3.0:1767319344.348505:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985146112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319344.348532:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985146112:12345-0@lo:400:kworker.0 Request processed in 28us (262us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319344.348538:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3001 00000100:00100000:3.0:1767319344.348582:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985146112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319344.348593:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985145472:0@lo:400:kworker.0 00000100:00100000:0.0:1767319344.348686:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985145984:0@lo:400:kworker.0 00000100:00100000:0.0:1767319344.348717:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319344.348741:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319344.348747:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985145728:0@lo:400:kworker.0 00000100:00100000:0.0:1767319344.348767:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319344.348788:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319344.348792:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319344.348801:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145728 00000100:00100000:0.0:1767319344.348804:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145728 00000100:00100000:0.0:1767319344.348833:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4563 00000100:00100000:0.0:1767319344.348840:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985145728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319344.348872:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985145728:12345-0@lo:400:kworker.0 Request processed in 33us (103us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319344.348879:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4563 00000100:00080000:0.0:1767319344.348893:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319344.348909:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985145728:0@lo:400:kworker.0 00000100:00100000:1.0:1767319344.349054:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985145216:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.349084:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.349111:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985145344:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.349132:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.349153:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985145600:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.349193:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.349209:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985145856:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.349227:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.349252:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145216 00000100:00100000:1.0:1767319344.349256:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145216 00000100:00100000:1.0:1767319344.349275:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145600 00000100:00100000:1.0:1767319344.349278:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145600 00000100:00100000:1.0:1767319344.349289:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145856 00000100:00100000:1.0:1767319344.349291:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145856 00000100:00100000:1.0:1767319344.349302:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5019 00000100:00100000:1.0:1767319344.349306:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985145216:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319344.349323:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319344.350194:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5020 00000100:00100000:0.0:1767319344.350201:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985145600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319344.350215:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319344.350279:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985145600:12345-0@lo:38:kworker.0 Request processed in 76us (1084us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319344.350286:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5020 00000100:00100000:0.0:1767319344.350312:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5021 00000100:00100000:0.0:1767319344.350316:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985145856:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319344.350325:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319344.350349:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985145856:12345-0@lo:38:kworker.0 Request processed in 32us (1121us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319344.350355:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5021 00000100:00080000:0.0:1767319344.350466:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319344.350472:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319344.350477:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985145600:0@lo:38:kworker.0 00000100:00080000:0.0:1767319344.350488:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319344.350491:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319344.350494:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985145856:0@lo:38:kworker.0 00000100:00100000:2.0:1767319344.350807:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145344 00000100:00100000:2.0:1767319344.350815:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145344 00000100:00100000:2.0:1767319344.350847:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1745 00000100:00100000:2.0:1767319344.350853:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985145344:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319344.350863:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319344.350903:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985145344:12345-0@lo:38:kworker.0 Request processed in 49us (1769us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319344.350909:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1745 00000100:00080000:2.0:1767319344.350947:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319344.350951:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319344.350954:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985145344:0@lo:38:kworker.0 00000100:00100000:3.0:1767319344.358658:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985145984 00000100:00100000:3.0:1767319344.358672:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985145984 00000100:00100000:3.0:1767319344.358900:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3002 00000100:00100000:3.0:1767319344.358910:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985145984:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319344.359008:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985145984:12345-0@lo:400:kworker.0 Request processed in 100us (10290us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319344.359016:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3002 00000100:00100000:3.0:1767319344.359066:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985145984:0@lo:400:kworker.0 00000100:00080000:2.0:1767319344.368559:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319344.392325:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985145216:12345-0@lo:38:kworker.0 Request processed in 43006us (43228us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319344.392341:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5019 00000100:00080000:2.0:1767319344.392754:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319344.392762:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319344.392767:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985145216:0@lo:38:kworker.0 00010000:00010000:2.0:1767319344.987488:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319345.324272:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319345.324370:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319345.324600:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920555520 00000100:00100000:3.0:1767319345.324610:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920555520 00000100:00100000:3.0:1767319345.324695:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920555392 00000100:00100000:3.0:1767319345.324698:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920555392 00000100:00100000:3.0:1767319345.324716:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4564 00000100:00100000:3.0:1767319345.324726:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920555520:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319345.324753:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5022 00000100:00100000:0.0:1767319345.324763:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920555392:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319345.324784:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319345.324801:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920555520:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (538us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319345.324809:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4564 00000100:00100000:0.0:1767319345.324821:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920555392:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (450us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319345.324830:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5022 00000100:00080000:2.0:1767319345.325374:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319345.883448:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985146240:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319345.883527:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319345.883626:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985146240 00000100:00100000:2.0:1767319345.883635:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985146240 00000100:00100000:2.0:1767319345.883691:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2168 00000100:00100000:2.0:1767319345.883702:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985146240:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319345.883883:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319345.884140:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985146368:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319345.884195:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319345.884267:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985146368 00000100:00100000:1.0:1767319345.884272:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985146368 00000100:00100000:1.0:1767319345.884298:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2169 00000100:00100000:1.0:1767319345.884304:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985146368:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319345.884319:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319345.884394:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985146240:12345-0@lo:13:osp-pre-0-1.0 Request processed in 685us (857us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319345.884402:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2168 00000100:00100000:3.0:1767319345.887928:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985146240:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319345.890076:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985146368:12345-0@lo:13:osp-pre-1-1.0 Request processed in 5763us (5872us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319345.890091:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2169 00000100:00100000:0.0:1767319345.892695:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985146368:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319346.011939:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319347.038980:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319348.059381:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319349.083758:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319349.467350:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319349.467360:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319349.467363:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319349.467367:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319349.467371:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319349.467381:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4592 00000100:00080000:0.0:1767319349.467385:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319349.467391:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319349.467429:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10700 x1853163985146496/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319349.467454:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319349.467458:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319349.467459:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319349.467463:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4592 00000100:00080000:0.0:1767319349.467465:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319349.467469:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319349.467483:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1f80 x1853163985146624/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319349.467529:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319349.467533:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319349.467536:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319349.467539:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4592 00000100:00080000:0.0:1767319349.467541:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319349.467545:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319349.467557:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985146880/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319349.467568:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767319349.467580:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319349.467586:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767319349.467586:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985146752:0@lo:400:kworker.0 00000100:00080000:0.0:1767319349.467589:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319349.467597:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4592 00000100:00080000:0.0:1767319349.467599:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319349.467602:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319349.467614:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1180 x1853163985147136/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319349.467654:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319349.467687:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985146752 00000100:00100000:0.0:1767319349.467695:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985146752 00000100:00100000:1.0:1767319349.467728:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985147392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319349.467758:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319349.467769:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4565 00000100:00100000:0.0:1767319349.467782:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985146752:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319349.467800:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319349.467837:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985146752:12345-0@lo:400:kworker.0 Request processed in 58us (182us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319349.467845:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4565 00000100:00100000:1.0:1767319349.467859:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985146752:0@lo:400:kworker.0 00000100:00100000:3.0:1767319349.467928:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147392 00000100:00100000:3.0:1767319349.467935:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147392 00000100:00080000:1.0:1767319349.467970:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319349.467982:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3003 00000100:00100000:3.0:1767319349.467990:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985147392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319349.467991:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985147264:0@lo:400:kworker.0 00000100:00100000:1.0:1767319349.468030:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319349.468046:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985147392:12345-0@lo:400:kworker.0 Request processed in 56us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319349.468054:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319349.468054:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3003 00000100:00100000:1.0:1767319349.468062:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985147008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319349.468070:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147264 00000100:00100000:3.0:1767319349.468074:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147264 00000100:00100000:3.0:1767319349.468099:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3004 00000100:00100000:3.0:1767319349.468104:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985147264:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319349.468132:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985147264:12345-0@lo:400:kworker.0 Request processed in 28us (102us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319349.468137:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3004 00000100:00080000:3.0:1767319349.468182:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319349.468206:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985147392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319349.468460:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319349.468528:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985147264:0@lo:400:kworker.0 00000100:00100000:0.0:1767319349.468564:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147008 00000100:00100000:0.0:1767319349.468568:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147008 00000100:00080000:3.0:1767319349.468619:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319349.468637:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4566 00000100:00100000:0.0:1767319349.468643:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985147008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319349.468672:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985147008:12345-0@lo:400:kworker.0 Request processed in 30us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319349.468677:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4566 00000100:00100000:3.0:1767319349.468711:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985147008:0@lo:400:kworker.0 00000100:00100000:2.0:1767319349.468749:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985146496:0@lo:38:kworker.0 00000100:00100000:2.0:1767319349.468781:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319349.468813:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985146496 00000100:00100000:2.0:1767319349.468814:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985146624:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.468817:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985146496 00000100:00100000:2.0:1767319349.468845:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319349.468861:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5023 00000100:00100000:0.0:1767319349.468865:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985146496:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319349.468880:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985146880:0@lo:38:kworker.0 00010000:00020000:0.0:1767319349.468882:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319349.468897:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985146624 00000100:00100000:3.0:1767319349.468903:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985146624 00000100:00100000:1.0:1767319349.469021:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1746 00000100:00100000:1.0:1767319349.469025:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985146624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319349.469032:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319349.470976:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319349.470983:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985146496:12345-0@lo:38:kworker.0 Request processed in 2113us (2199us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319349.470992:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5023 00000100:00100000:0.0:1767319349.471013:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985146880 00000100:00100000:0.0:1767319349.471019:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985146880 00000100:00100000:2.0:1767319349.471029:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985147136:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.471053:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5024 00000100:00100000:0.0:1767319349.471058:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985146880:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319349.471065:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767319349.471075:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319349.471106:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319349.471114:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985146880:12345-0@lo:38:kworker.0 Request processed in 53us (143us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319349.471114:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319349.471122:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985146496:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.471127:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5024 00000100:00080000:2.0:1767319349.471140:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319349.471142:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147136 00000100:00080000:2.0:1767319349.471146:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319349.471147:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147136 00000100:00100000:2.0:1767319349.471152:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985146880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.471190:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5025 00000100:00100000:0.0:1767319349.471196:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985147136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319349.471359:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319349.471472:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985147136:12345-0@lo:38:kworker.0 Request processed in 274us (407us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319349.471478:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5025 00000100:00080000:2.0:1767319349.471552:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319349.471556:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319349.471559:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985147136:0@lo:38:kworker.0 00000100:00100000:1.0:1767319349.471816:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985146624:12345-0@lo:38:kworker.0 Request processed in 2787us (2969us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319349.471824:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1746 00000100:00080000:2.0:1767319349.471876:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319349.471880:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319349.471884:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985146624:0@lo:38:kworker.0 00010000:00010000:1.0:1767319350.107452:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319350.442486:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319350.442943:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920555648 00000100:00100000:3.0:1767319350.442953:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920555648 00000100:00100000:3.0:1767319350.443008:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5026 00000100:00100000:3.0:1767319350.443016:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920555648:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319350.443038:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319350.443106:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920555648:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 89us (630us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319350.443114:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5026 00000100:00100000:2.0:1767319350.443215:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319350.443521:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920555776 00000100:00100000:0.0:1767319350.443526:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920555776 00000100:00100000:0.0:1767319350.443582:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4567 00000100:00100000:0.0:1767319350.443599:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920555776:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319350.443654:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920555776:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 64us (439us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319350.443660:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4567 00000100:00080000:1.0:1767319350.443852:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319351.003512:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985147520:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319351.003597:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319351.003743:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147520 00000100:00100000:0.0:1767319351.003751:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147520 00000100:00100000:0.0:1767319351.003810:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2170 00000100:00100000:0.0:1767319351.003819:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985147520:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319351.003847:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319351.003885:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985147520:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319351.003892:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2170 00000100:00080000:0.0:1767319351.003913:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319351.004258:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985147520:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319351.004279:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319351.004287:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985147648:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319351.004340:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319351.007767:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147648 00000100:00100000:2.0:1767319351.007779:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147648 00000100:00100000:2.0:1767319351.007924:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2171 00000100:00100000:2.0:1767319351.007934:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985147648:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319351.007968:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319351.008053:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985147648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 122us (3712us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319351.008061:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2171 00000100:00080000:0.0:1767319351.008405:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319351.009298:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985147648:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319351.134460:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319352.155433:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319353.180044:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319354.203754:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319354.587911:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319354.587923:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319354.587926:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319354.587930:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319354.587934:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319354.587943:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4597 00000100:00080000:1.0:1767319354.587948:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319354.587955:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319354.587996:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b800 x1853163985147776/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319354.588020:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319354.588024:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319354.588026:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319354.588029:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4597 00000100:00080000:1.0:1767319354.588031:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319354.588035:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319354.588047:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579f80 x1853163985147904/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319354.588085:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319354.588088:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319354.588090:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319354.588093:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4597 00000100:00080000:1.0:1767319354.588096:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319354.588098:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319354.588110:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357aa00 x1853163985148160/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319354.588128:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985148032:0@lo:400:kworker.0 00000100:00080000:1.0:1767319354.588138:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319354.588142:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319354.588144:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319354.588147:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4597 00000100:00080000:1.0:1767319354.588150:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319354.588154:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319354.588194:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578700 x1853163985148416/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319354.588234:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319354.588254:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985148288:0@lo:400:kworker.0 00000100:00100000:1.0:1767319354.588287:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319354.588299:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319354.588309:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985148672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319354.588315:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319354.588323:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985148544:0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.588362:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319354.588375:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148032 00000100:00100000:1.0:1767319354.588383:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148032 00000100:00100000:2.0:1767319354.588403:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148288 00000100:00100000:2.0:1767319354.588409:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148288 00000100:00100000:2.0:1767319354.588464:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4568 00000100:00100000:2.0:1767319354.588477:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985148288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319354.588491:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148672 00000100:00100000:0.0:1767319354.588496:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148672 00000100:00100000:0.0:1767319354.588524:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3005 00000100:00100000:0.0:1767319354.588531:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985148672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.588531:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985148288:12345-0@lo:400:kworker.0 Request processed in 56us (242us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319354.588545:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4568 00000100:00100000:0.0:1767319354.588569:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985148672:12345-0@lo:400:kworker.0 Request processed in 40us (208us total) trans 0 rc 0/0 00000100:00080000:2.0:1767319354.588573:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319354.588574:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3005 00000100:00080000:2.0:1767319354.588581:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319354.588620:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985148672:0@lo:400:kworker.0 00000100:00100000:0.0:1767319354.589472:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319354.589507:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985148288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.589516:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148544 00000100:00100000:2.0:1767319354.589523:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148544 00000100:00100000:2.0:1767319354.589572:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3006 00000100:00100000:2.0:1767319354.589581:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985148544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.589628:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985148544:12345-0@lo:400:kworker.0 Request processed in 48us (1008us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319354.589635:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3006 00000100:00100000:2.0:1767319354.589669:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985148544:0@lo:400:kworker.0 00000100:00100000:1.0:1767319354.589680:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4569 00000100:00100000:1.0:1767319354.589686:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985148032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319354.589729:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985148032:12345-0@lo:400:kworker.0 Request processed in 44us (1509us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319354.589733:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4569 00000100:00100000:2.0:1767319354.589773:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985148032:0@lo:400:kworker.0 00000100:00080000:0.0:1767319354.589778:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319354.589783:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319354.590485:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985147776:0@lo:38:kworker.0 00000100:00100000:3.0:1767319354.590524:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319354.590561:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985147904:0@lo:38:kworker.0 00000100:00100000:3.0:1767319354.590589:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319354.590620:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985148160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319354.590640:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147904 00000100:00100000:3.0:1767319354.590643:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319354.590647:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147904 00000100:00100000:3.0:1767319354.590662:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985148416:0@lo:38:kworker.0 00000100:00100000:2.0:1767319354.590679:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1747 00000100:00100000:2.0:1767319354.590689:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985147904:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319354.590689:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767319354.590711:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319354.590734:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985147776 00000100:00100000:3.0:1767319354.590738:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985147776 00000100:00100000:2.0:1767319354.590751:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985147904:12345-0@lo:38:kworker.0 Request processed in 63us (160us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319354.590755:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148160 00000100:00100000:3.0:1767319354.590758:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148160 00000100:00100000:2.0:1767319354.590759:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1747 00000100:00100000:3.0:1767319354.590769:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148416 00000100:00100000:3.0:1767319354.590771:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148416 00000100:00100000:3.0:1767319354.590785:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5027 00000100:00100000:3.0:1767319354.590788:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985147776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319354.590797:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319354.590814:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5028 00000100:00100000:2.0:1767319354.590820:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985148160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319354.590830:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319354.590857:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985148160:12345-0@lo:38:kworker.0 Request processed in 36us (215us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319354.590863:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5028 00000100:00100000:2.0:1767319354.590877:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5029 00000100:00100000:2.0:1767319354.590880:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985148416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319354.590889:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319354.590913:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985148416:12345-0@lo:38:kworker.0 Request processed in 31us (228us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319354.590919:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5029 00000100:00080000:2.0:1767319354.590947:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.590952:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.590955:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985147904:0@lo:38:kworker.0 00000100:00080000:2.0:1767319354.590963:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.590966:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.590969:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985148160:0@lo:38:kworker.0 00000100:00080000:2.0:1767319354.590974:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.590977:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.590979:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985148416:0@lo:38:kworker.0 00000100:00100000:3.0:1767319354.591283:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985147776:12345-0@lo:38:kworker.0 Request processed in 493us (759us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319354.591288:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5027 00000100:00080000:2.0:1767319354.591439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.591445:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.591448:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985147776:0@lo:38:kworker.0 00010000:00010000:0.0:1767319355.227495:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319355.564729:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319355.564781:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319355.565048:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920556032 00000100:00100000:3.0:1767319355.565058:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920556032 00000100:00100000:3.0:1767319355.565133:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920555904 00000100:00100000:3.0:1767319355.565136:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920555904 00000100:00100000:3.0:1767319355.565157:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4570 00000100:00100000:3.0:1767319355.565196:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920556032:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767319355.565289:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319355.565306:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920556032:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 139us (586us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319355.565315:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4570 00000100:00100000:3.0:1767319355.565332:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5030 00000100:00100000:3.0:1767319355.565337:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920555904:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319355.565357:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319355.565380:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920555904:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (599us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319355.565386:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5030 00000100:00100000:1.0:1767319356.126094:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319356.126116:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985148800:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319356.126219:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319356.126324:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148800 00000100:00100000:3.0:1767319356.126333:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148800 00000100:00100000:3.0:1767319356.126406:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2172 00000100:00100000:3.0:1767319356.126423:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985148800:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319356.126453:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319356.126498:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985148800:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (278us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319356.126505:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2172 00000100:00100000:0.0:1767319356.126527:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985148928:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319356.126563:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319356.126607:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319356.126658:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985148800:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319356.126673:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985148928 00000100:00100000:3.0:1767319356.126679:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985148928 00000100:00080000:1.0:1767319356.126705:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319356.126719:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2173 00000100:00100000:3.0:1767319356.126727:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985148928:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319356.126751:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319356.130500:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985148928:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3764us (3926us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319356.130514:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2173 00000100:00100000:0.0:1767319356.130666:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985148928:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319356.252703:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319357.275407:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319358.299689:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319359.323362:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319359.707592:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.707609:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319359.707613:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.707617:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.707623:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.707650:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.707656:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.707664:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319359.707719:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1c00 x1853163985149056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319359.707775:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985149056:0@lo:38:kworker.0 00000100:00100000:2.0:1767319359.707853:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319359.708316:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.708322:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.708324:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.708329:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.708333:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.708337:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319359.708363:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2680 x1853163985149184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319359.708442:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149056 00000100:00100000:1.0:1767319359.708453:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149056 00000100:00100000:1.0:1767319359.708517:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5031 00000100:00100000:1.0:1767319359.708527:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985149056:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319359.708553:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319359.709050:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985149184:0@lo:38:kworker.0 00000100:00100000:2.0:1767319359.709093:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319359.709252:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149184 00000100:00100000:0.0:1767319359.709259:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149184 00000100:00100000:0.0:1767319359.709304:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1748 00000100:00100000:0.0:1767319359.709311:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985149184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319359.709327:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319359.709364:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985149184:12345-0@lo:38:kworker.0 Request processed in 52us (270us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319359.709372:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1748 00000100:00080000:3.0:1767319359.709498:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.709502:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.709505:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.709510:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.709513:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.709517:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319359.709533:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985149440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319359.709564:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.709567:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.709570:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.709574:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.709577:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.709581:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319359.709592:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0380 x1853163985149696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319359.709781:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985149056:12345-0@lo:38:kworker.0 Request processed in 1254us (1927us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319359.709790:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5031 00000100:00100000:1.0:1767319359.709821:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985149568:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.709851:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319359.709882:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319359.709888:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985149824:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.709914:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319359.710208:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149824 00000100:00100000:3.0:1767319359.710215:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149824 00000100:00100000:3.0:1767319359.710314:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3007 00000100:00100000:3.0:1767319359.710323:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985149824:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319359.710345:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319359.710354:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319359.710359:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985149056:0@lo:38:kworker.0 00000100:00100000:3.0:1767319359.710382:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985149824:12345-0@lo:400:kworker.0 Request processed in 60us (466us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319359.710392:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985149440:0@lo:38:kworker.0 00000100:00100000:3.0:1767319359.710392:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3007 00000100:00100000:3.0:1767319359.710420:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149568 00000100:00100000:3.0:1767319359.710425:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149568 00000100:00080000:1.0:1767319359.710463:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319359.710468:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319359.710512:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4571 00000100:00100000:3.0:1767319359.710518:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985149568:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319359.710694:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985149952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.711023:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985149824:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.711073:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985149312:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.711107:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319359.711198:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149312 00000100:00100000:1.0:1767319359.711203:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149312 00000100:00100000:1.0:1767319359.711253:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4572 00000100:00100000:1.0:1767319359.711260:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985149312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.711300:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985149312:12345-0@lo:400:kworker.0 Request processed in 40us (192us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319359.711306:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4572 00000100:00100000:1.0:1767319359.711339:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985149312:0@lo:400:kworker.0 00000100:00080000:1.0:1767319359.711354:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319359.711900:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985149568:12345-0@lo:400:kworker.0 Request processed in 1378us (2045us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319359.711908:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4571 00000100:00100000:1.0:1767319359.712018:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985149568:0@lo:400:kworker.0 00000100:00100000:0.0:1767319359.712079:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319359.712124:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149952 00000100:00100000:3.0:1767319359.712146:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149952 00000100:00100000:3.0:1767319359.712256:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3008 00000100:00100000:3.0:1767319359.712263:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985149952:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319359.712294:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985149952:12345-0@lo:400:kworker.0 Request processed in 33us (218us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319359.712298:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3008 00000100:00080000:1.0:1767319359.712330:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319359.712355:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985149952:0@lo:400:kworker.0 00000100:00100000:2.0:1767319359.714433:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319359.714492:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985149696:0@lo:38:kworker.0 00000100:00100000:2.0:1767319359.714536:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319359.714587:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319359.714682:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319359.714687:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985149184:0@lo:38:kworker.0 00000100:00100000:3.0:1767319359.714963:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149440 00000100:00100000:3.0:1767319359.714974:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149440 00000100:00100000:1.0:1767319359.715020:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985149696 00000100:00100000:1.0:1767319359.715026:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985149696 00000100:00100000:3.0:1767319359.715037:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5032 00000100:00100000:3.0:1767319359.715046:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985149440:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319359.715083:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319359.715128:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5033 00000100:00100000:1.0:1767319359.715145:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985149696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319359.715158:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319359.715159:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985149440:12345-0@lo:38:kworker.0 Request processed in 111us (735us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319359.715197:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5032 00000100:00100000:1.0:1767319359.715229:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985149696:12345-0@lo:38:kworker.0 Request processed in 82us (692us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319359.715236:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5033 00000100:00080000:2.0:1767319359.715305:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319359.715312:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319359.715317:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985149440:0@lo:38:kworker.0 00000100:00080000:2.0:1767319359.715328:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319359.715332:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319359.715334:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985149696:0@lo:38:kworker.0 00010000:00010000:0.0:1767319360.347530:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319360.683135:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319360.683238:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920556288 00000100:00100000:2.0:1767319360.683255:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920556288 00000100:00100000:2.0:1767319360.683546:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4573 00000100:00100000:2.0:1767319360.683560:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920556288:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319360.683652:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319360.683684:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920556288:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 125us (573us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319360.683694:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4573 00000100:00100000:1.0:1767319360.691124:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319360.691328:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920556160 00000100:00100000:0.0:1767319360.691362:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920556160 00000100:00100000:0.0:1767319360.691441:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5034 00000100:00100000:0.0:1767319360.691456:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920556160:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319360.691500:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319360.691609:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920556160:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 155us (505us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319360.691619:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5034 00000100:00100000:1.0:1767319361.243453:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985150080:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319361.243540:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319361.243545:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319361.243572:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985150208:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319361.243640:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985150080 00000100:00100000:0.0:1767319361.243648:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319361.243653:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985150080 00000100:00100000:3.0:1767319361.243719:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985150208 00000100:00100000:3.0:1767319361.243724:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985150208 00000100:00080000:0.0:1767319361.243741:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319361.243748:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319361.243764:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2174 00000100:00100000:3.0:1767319361.243777:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985150080:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319361.243778:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2175 00000100:00100000:0.0:1767319361.243798:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985150208:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319361.243816:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:0.0:1767319361.243833:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319361.244802:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985150208:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1005us (1151us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319361.244814:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2175 00000100:00100000:3.0:1767319361.245669:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985150080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1887us (2120us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319361.245684:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2174 00000100:00100000:0.0:1767319361.245729:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985150080:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319361.246456:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985150208:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319361.374992:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319362.396373:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319363.419428:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319364.443507:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319364.777733:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319364.777770:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920556416 offset 0 length 520: 4 00000100:00100000:3.0:1767319364.777787:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319364.777886:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920556672 00000100:00100000:2.0:1767319364.777896:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920556672 00000100:00100000:2.0:1767319364.777957:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920556544 00000100:00100000:2.0:1767319364.777960:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920556544 00000100:00100000:2.0:1767319364.777978:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4574 00000100:00100000:2.0:1767319364.777984:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920556672:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319364.778016:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5035 00000100:00100000:1.0:1767319364.778025:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920556544:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319364.778073:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319364.778075:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920556672:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (349us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319364.778084:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4574 00000100:00100000:1.0:1767319364.778107:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920556544:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 81us (316us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319364.778115:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5035 00000100:00080000:3.0:1767319364.778365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319364.827447:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319339/real 1767319339] req@ffff8ce1b34d1c00 x1853163985143808/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319364 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319364.827494:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b34d1c00 x1853163985143808/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319364 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319364.827509:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319364.827536:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319364.827540:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985143808:0@lo:250:kworker.0 00000100:00080000:0.0:1767319364.827717:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319364.827724:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319364.827727:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319364.827732:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4587 00000100:00080000:0.0:1767319364.827736:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319364.827743:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319364.827780:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2300 x1853163985150336/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319364.827804:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319364.827808:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319364.827810:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319364.827851:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4607 00000100:00080000:0.0:1767319364.827854:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319364.827857:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319364.827872:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1880 x1853163985150464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319364.827882:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319364.827884:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319364.827886:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319364.827889:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4607 00000100:00080000:0.0:1767319364.827892:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319364.827895:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319364.827907:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0a80 x1853163985150592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319364.827950:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319364.827953:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319364.827955:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319364.827959:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4607 00000100:00080000:0.0:1767319364.827961:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319364.827965:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319364.827979:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630000 x1853163985150848/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319364.828018:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319364.828021:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319364.828023:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319364.828027:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4607 00000100:00080000:0.0:1767319364.828029:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319364.828032:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319364.828053:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633800 x1853163985151104/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319364.828091:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985150976:0@lo:400:kworker.0 00000100:00100000:0.0:1767319364.828147:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319364.828228:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319364.828235:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985151232:0@lo:400:kworker.0 00000100:00100000:0.0:1767319364.828260:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319364.828294:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985150976 00000100:00100000:0.0:1767319364.828300:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985150976 00000100:00100000:0.0:1767319364.828351:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4575 00000100:00100000:0.0:1767319364.828361:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985150976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319364.828410:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985150976:12345-0@lo:400:kworker.0 Request processed in 50us (262us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319364.828417:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4575 00000100:00080000:0.0:1767319364.828439:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319364.828481:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985150976:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829397:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985151232 00000100:00100000:3.0:1767319364.829404:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985151232 00000100:00100000:3.0:1767319364.829441:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3009 00000100:00100000:3.0:1767319364.829448:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985151232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829508:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985151232:12345-0@lo:400:kworker.0 Request processed in 61us (1246us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319364.829515:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3009 00000100:00100000:3.0:1767319364.829539:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985151360:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829565:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319364.829586:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319364.829592:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985150720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829614:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319364.829644:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985151360 00000100:00100000:3.0:1767319364.829648:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985151360 00000100:00100000:3.0:1767319364.829673:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3010 00000100:00100000:3.0:1767319364.829679:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985151360:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829712:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985151360:12345-0@lo:400:kworker.0 Request processed in 33us (146us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319364.829718:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3010 00000100:00100000:3.0:1767319364.829750:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985151360:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829772:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985150720 00000100:00100000:3.0:1767319364.829775:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985150720 00000100:00100000:3.0:1767319364.829801:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4576 00000100:00100000:3.0:1767319364.829808:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985150720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829838:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985150720:12345-0@lo:400:kworker.0 Request processed in 31us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319364.829844:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4576 00000100:00100000:3.0:1767319364.829866:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985150720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.829894:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985151232:0@lo:400:kworker.0 00000100:00100000:2.0:1767319364.832154:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985150336:0@lo:250:kworker.0 00000400:00000100:2.0:1767319364.832230:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985150336 offset 0 length 520: 4 00000100:00100000:2.0:1767319364.832268:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985150464:0@lo:38:kworker.0 00000100:00100000:2.0:1767319364.832299:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319364.832341:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985150592:0@lo:38:kworker.0 00000100:00100000:2.0:1767319364.832367:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319364.832396:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985150848:0@lo:38:kworker.0 00000100:00100000:2.0:1767319364.832417:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319364.832436:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985151104:0@lo:38:kworker.0 00000100:00100000:2.0:1767319364.832465:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319364.832504:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985150464 00000100:00100000:2.0:1767319364.832512:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985150464 00000100:00100000:2.0:1767319364.832541:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985150848 00000100:00100000:2.0:1767319364.832544:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985150848 00000100:00100000:2.0:1767319364.832556:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985151104 00000100:00100000:2.0:1767319364.832558:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985151104 00000100:00100000:2.0:1767319364.832575:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5036 00000100:00100000:2.0:1767319364.832583:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985150464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319364.832605:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319364.832645:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985150464:12345-0@lo:38:kworker.0 Request processed in 61us (345us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319364.832656:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5036 00000100:00100000:2.0:1767319364.832672:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5037 00000100:00100000:2.0:1767319364.832676:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985150848:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319364.832684:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319364.832707:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985150848:12345-0@lo:38:kworker.0 Request processed in 30us (289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319364.832713:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5037 00000100:00100000:2.0:1767319364.832723:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5038 00000100:00100000:2.0:1767319364.832727:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985151104:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319364.832735:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319364.832756:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985151104:12345-0@lo:38:kworker.0 Request processed in 27us (289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319364.832760:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5038 00000100:00080000:2.0:1767319364.832804:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319364.832809:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319364.832813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985150464:0@lo:38:kworker.0 00000100:00080000:2.0:1767319364.832821:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319364.832824:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319364.832827:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985150848:0@lo:38:kworker.0 00000100:00080000:2.0:1767319364.832833:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319364.832836:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319364.832838:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985151104:0@lo:38:kworker.0 00000100:00100000:1.0:1767319364.832957:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985150592 00000100:00100000:1.0:1767319364.832962:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985150592 00000100:00100000:1.0:1767319364.832998:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1749 00000100:00100000:1.0:1767319364.833003:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985150592:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319364.833013:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319364.833064:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985150592:12345-0@lo:38:kworker.0 Request processed in 60us (696us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319364.833070:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1749 00000100:00080000:1.0:1767319364.833094:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319364.833097:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319364.833100:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985150592:0@lo:38:kworker.0 00000100:00080000:0.0:1767319364.833471:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319364.833478:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319364.833481:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319365.467569:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319366.363420:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985151488:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319366.363494:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319366.363582:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985151488 00000100:00100000:3.0:1767319366.363590:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985151488 00000100:00100000:3.0:1767319366.363650:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2176 00000100:00100000:3.0:1767319366.363656:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985151488:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319366.363681:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319366.363715:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985151488:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (221us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319366.363719:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2176 00000100:00080000:1.0:1767319366.363764:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319366.363793:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985151488:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319366.364738:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985151616:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319366.364777:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319366.365087:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985151616 00000100:00100000:0.0:1767319366.365094:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985151616 00000100:00080000:1.0:1767319366.365186:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319366.365236:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2177 00000100:00100000:0.0:1767319366.365245:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985151616:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319366.365269:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319366.365520:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985151616:12345-0@lo:13:osp-pre-1-1.0 Request processed in 276us (741us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319366.365528:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2177 00000100:00100000:3.0:1767319366.365591:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985151616:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319366.491534:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319367.515882:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319368.540487:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319369.563476:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319369.898483:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319369.898566:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319369.898845:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920556928 00000100:00100000:0.0:1767319369.898855:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920556928 00000100:00100000:0.0:1767319369.898929:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920556800 00000100:00100000:0.0:1767319369.898932:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920556800 00000100:00100000:0.0:1767319369.898961:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4577 00000100:00100000:0.0:1767319369.898972:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920556928:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767319369.898989:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319369.899050:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920556928:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (575us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319369.899058:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4577 00000100:00100000:0.0:1767319369.899077:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5039 00000100:00100000:0.0:1767319369.899081:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920556800:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319369.899100:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319369.899124:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920556800:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (559us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319369.899130:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5039 00000100:00080000:3.0:1767319369.947566:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319369.947576:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319369.947579:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319369.947583:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319369.947588:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319369.947596:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4613 00000100:00080000:3.0:1767319369.947600:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319369.947605:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319369.947648:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7480 x1853163985151744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319369.947672:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319369.947676:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319369.947678:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319369.947682:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4613 00000100:00080000:3.0:1767319369.947684:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319369.947687:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319369.947699:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4e00 x1853163985151872/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319369.947743:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319369.947747:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319369.947749:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319369.947753:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4613 00000100:00080000:3.0:1767319369.947756:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319369.947758:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319369.947770:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5500 x1853163985152128/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319369.947794:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319369.947798:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319369.947800:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319369.947803:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4613 00000100:00080000:3.0:1767319369.947805:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319369.947809:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319369.947819:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5180 x1853163985152384/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319369.947854:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985152256:0@lo:400:kworker.0 00000100:00100000:3.0:1767319369.948000:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319369.948056:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319369.948062:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985152512:0@lo:400:kworker.0 00000100:00100000:1.0:1767319369.948070:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985151744:0@lo:38:kworker.0 00000100:00100000:3.0:1767319369.948088:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319369.948093:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319369.948113:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985151872:0@lo:38:kworker.0 00000100:00100000:3.0:1767319369.948118:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152512 00000100:00100000:3.0:1767319369.948124:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152512 00000100:00100000:1.0:1767319369.948134:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319369.948150:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985152128:0@lo:38:kworker.0 00000100:00100000:1.0:1767319369.948188:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319369.948190:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3011 00000100:00100000:3.0:1767319369.948199:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985152512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319369.948204:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985152384:0@lo:38:kworker.0 00000100:00100000:1.0:1767319369.948221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319369.948261:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152256 00000100:00100000:1.0:1767319369.948264:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152256 00000100:00100000:1.0:1767319369.948288:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985151744 00000100:00100000:1.0:1767319369.948291:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985151744 00000100:00100000:1.0:1767319369.948302:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152128 00000100:00100000:1.0:1767319369.948305:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152128 00000100:00100000:1.0:1767319369.948312:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152384 00000100:00100000:1.0:1767319369.948313:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152384 00000100:00100000:1.0:1767319369.948322:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4578 00000100:00100000:1.0:1767319369.948329:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985152256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.948348:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985151872 00000100:00100000:2.0:1767319369.948364:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985151872 00000100:00100000:2.0:1767319369.948391:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1750 00000100:00100000:2.0:1767319369.948396:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985151872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319369.948414:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319369.948483:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985152000:0@lo:400:kworker.0 00000100:00100000:1.0:1767319369.956804:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985152256:12345-0@lo:400:kworker.0 Request processed in 8464us (8793us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319369.956819:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4578 00000100:00100000:1.0:1767319369.957160:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5040 00000100:00100000:1.0:1767319369.957185:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985151744:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319369.957206:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319369.957266:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985151744:12345-0@lo:38:kworker.0 Request processed in 80us (9171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319369.957273:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5040 00000100:00100000:1.0:1767319369.957286:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5041 00000100:00100000:1.0:1767319369.957291:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985152128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319369.957300:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319369.957323:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985152128:12345-0@lo:38:kworker.0 Request processed in 30us (9134us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319369.957328:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5041 00000100:00100000:1.0:1767319369.957338:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5042 00000100:00100000:1.0:1767319369.957342:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985152384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319369.957350:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319369.957373:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985152384:12345-0@lo:38:kworker.0 Request processed in 30us (9150us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319369.957379:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5042 00000100:00080000:1.0:1767319369.957401:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319369.957406:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319369.957439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319369.957444:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319369.957448:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985151744:0@lo:38:kworker.0 00000100:00080000:1.0:1767319369.957458:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319369.957462:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319369.957465:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985152128:0@lo:38:kworker.0 00000100:00080000:1.0:1767319369.957473:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319369.957476:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319369.957479:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985152384:0@lo:38:kworker.0 00000100:00100000:1.0:1767319369.957571:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985152640:0@lo:400:kworker.0 00000100:00100000:1.0:1767319369.957606:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319369.957650:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152640 00000100:00100000:1.0:1767319369.957656:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152640 00000100:00100000:1.0:1767319369.957693:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3012 00000100:00100000:1.0:1767319369.957700:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985152640:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319369.957737:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985152640:12345-0@lo:400:kworker.0 Request processed in 37us (131us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319369.957743:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3012 00000100:00080000:1.0:1767319369.957759:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319369.957781:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985152256:0@lo:400:kworker.0 00000100:00100000:1.0:1767319369.957802:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985152640:0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.967023:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985151872:12345-0@lo:38:kworker.0 Request processed in 18616us (18877us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319369.967037:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1750 00000100:00080000:1.0:1767319369.968153:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319369.968190:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319369.968196:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985151872:0@lo:38:kworker.0 00000100:00100000:3.0:1767319369.971624:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985152512:12345-0@lo:400:kworker.0 Request processed in 23415us (23523us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319369.971640:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3011 00000100:00100000:1.0:1767319369.971789:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985152512:0@lo:400:kworker.0 00000100:00100000:0.0:1767319369.971927:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319369.972024:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152000 00000100:00100000:1.0:1767319369.972031:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152000 00000100:00100000:1.0:1767319369.972102:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4579 00000100:00100000:1.0:1767319369.972110:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985152000:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319369.972189:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985152000:12345-0@lo:400:kworker.0 Request processed in 80us (262us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319369.972195:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4579 00000100:00080000:2.0:1767319369.972259:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319369.972509:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985152000:0@lo:400:kworker.0 00010000:00010000:2.0:1767319370.587878:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319371.483778:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985152768:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319371.483849:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319371.483970:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152768 00000100:00100000:0.0:1767319371.483979:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152768 00000100:00100000:0.0:1767319371.484043:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2178 00000100:00100000:0.0:1767319371.484053:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985152768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319371.484082:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319371.484125:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985152768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319371.484132:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2178 00000100:00100000:0.0:1767319371.484359:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985152896:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319371.484387:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319371.484551:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985152896 00000100:00100000:2.0:1767319371.484556:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985152896 00000100:00100000:2.0:1767319371.484593:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2179 00000100:00100000:2.0:1767319371.484602:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985152896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319371.484620:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319371.484656:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985152896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (268us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319371.484663:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2179 00000100:00080000:1.0:1767319371.484774:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319371.484778:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319371.484815:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985152768:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319371.484995:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985152896:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319371.615070:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319372.635868:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319373.660675:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319374.687217:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319375.017976:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319375.018020:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319375.018155:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920557184 00000100:00100000:2.0:1767319375.018189:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920557184 00000100:00100000:2.0:1767319375.018278:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920557056 00000100:00100000:2.0:1767319375.018282:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920557056 00000100:00100000:2.0:1767319375.018298:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4580 00000100:00100000:2.0:1767319375.018308:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920557184:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319375.018371:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920557184:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 66us (416us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319375.018381:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4580 00000100:00100000:2.0:1767319375.018397:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5043 00000100:00100000:2.0:1767319375.018401:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920557056:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319375.018418:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319375.018440:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920557056:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 38us (419us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319375.018447:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5043 00000100:00080000:1.0:1767319375.018592:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319375.069304:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319375.069316:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319375.069319:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319375.069323:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319375.069327:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319375.069335:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4618 00000100:00080000:0.0:1767319375.069339:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319375.069346:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319375.069385:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631880 x1853163985153024/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319375.069411:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319375.069415:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319375.069416:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319375.069420:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4618 00000100:00080000:0.0:1767319375.069422:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319375.069425:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319375.069438:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630380 x1853163985153152/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319375.069475:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319375.069478:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319375.069480:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319375.069484:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4618 00000100:00080000:0.0:1767319375.069486:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319375.069489:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319375.069500:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632300 x1853163985153408/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319375.069522:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319375.069526:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319375.069528:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319375.069531:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4618 00000100:00080000:0.0:1767319375.069534:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319375.069537:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319375.069549:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632a00 x1853163985153664/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319375.069560:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985153536:0@lo:400:kworker.0 00000100:00100000:0.0:1767319375.069606:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985153920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319375.069632:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319375.069642:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319375.069696:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319375.069704:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985153792:0@lo:400:kworker.0 00000100:00100000:3.0:1767319375.069730:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319375.069735:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153536 00000100:00100000:1.0:1767319375.069743:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153536 00000100:00100000:3.0:1767319375.069760:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153920 00000100:00100000:3.0:1767319375.069766:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153920 00000100:00100000:1.0:1767319375.069798:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4581 00000100:00100000:3.0:1767319375.069802:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153792 00000100:00100000:3.0:1767319375.069806:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153792 00000100:00100000:1.0:1767319375.069811:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985153536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319375.069862:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3013 00000100:00100000:3.0:1767319375.069870:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985153920:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319375.069877:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985153536:12345-0@lo:400:kworker.0 Request processed in 69us (244us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319375.069886:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4581 00000100:00100000:1.0:1767319375.069915:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3014 00000100:00100000:3.0:1767319375.069915:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985153920:12345-0@lo:400:kworker.0 Request processed in 44us (270us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319375.069927:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985153792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319375.069928:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3013 00000100:00100000:3.0:1767319375.069957:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985153280:0@lo:400:kworker.0 00000100:00100000:1.0:1767319375.069963:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985153792:12345-0@lo:400:kworker.0 Request processed in 39us (232us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319375.069970:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3014 00000100:00100000:3.0:1767319375.069986:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319375.069994:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319375.070000:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319375.070003:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319375.070016:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153280 00000100:00100000:1.0:1767319375.070020:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153280 00000100:00100000:1.0:1767319375.070045:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4582 00000100:00100000:1.0:1767319375.070050:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985153280:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319375.070085:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985153280:12345-0@lo:400:kworker.0 Request processed in 34us (100us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319375.070090:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4582 00000100:00080000:1.0:1767319375.070105:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319375.070112:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985153280:0@lo:400:kworker.0 00000100:00100000:1.0:1767319375.070130:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985153792:0@lo:400:kworker.0 00000100:00100000:1.0:1767319375.070142:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985153536:0@lo:400:kworker.0 00000100:00100000:2.0:1767319375.070248:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985153024:0@lo:38:kworker.0 00000100:00100000:2.0:1767319375.070277:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319375.070306:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985153152:0@lo:38:kworker.0 00000100:00100000:2.0:1767319375.070330:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319375.070355:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985153408:0@lo:38:kworker.0 00000100:00100000:2.0:1767319375.070372:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319375.070384:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985153664:0@lo:38:kworker.0 00000100:00100000:2.0:1767319375.070398:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319375.070420:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153024 00000100:00100000:2.0:1767319375.070423:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153024 00000100:00100000:2.0:1767319375.070438:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153408 00000100:00100000:2.0:1767319375.070440:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153408 00000100:00100000:2.0:1767319375.070449:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153664 00000100:00100000:2.0:1767319375.070452:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153664 00000100:00100000:2.0:1767319375.070466:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5044 00000100:00100000:2.0:1767319375.070470:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985153024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319375.070485:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319375.070512:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985153024:12345-0@lo:38:kworker.0 Request processed in 41us (235us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319375.070518:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5044 00000100:00100000:2.0:1767319375.070530:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5045 00000100:00100000:2.0:1767319375.070534:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985153408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319375.070542:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319375.070561:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985153408:12345-0@lo:38:kworker.0 Request processed in 25us (188us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319375.070567:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985153152 00000100:00100000:2.0:1767319375.070568:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5045 00000100:00100000:3.0:1767319375.070573:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985153152 00000100:00100000:2.0:1767319375.070581:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5046 00000100:00100000:2.0:1767319375.070586:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985153664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319375.070595:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319375.070602:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1751 00000100:00100000:3.0:1767319375.070609:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985153152:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319375.070616:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985153664:12345-0@lo:38:kworker.0 Request processed in 27us (215us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319375.070622:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5046 00010000:00020000:3.0:1767319375.070625:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319375.070648:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319375.070654:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319375.070658:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985153024:0@lo:38:kworker.0 00000100:00100000:3.0:1767319375.070664:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985153152:12345-0@lo:38:kworker.0 Request processed in 53us (331us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319375.070669:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319375.070674:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1751 00000100:00080000:2.0:1767319375.070676:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319375.070681:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985153152:0@lo:38:kworker.0 00000100:00080000:2.0:1767319375.070691:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319375.070694:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319375.070698:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985153408:0@lo:38:kworker.0 00000100:00080000:2.0:1767319375.070706:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319375.070709:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319375.070713:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985153664:0@lo:38:kworker.0 00000100:00100000:0.0:1767319375.073646:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985153920:0@lo:400:kworker.0 00010000:00010000:2.0:1767319375.707827:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319376.604804:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985154048:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319376.604885:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319376.604977:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154048 00000100:00100000:0.0:1767319376.604986:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154048 00000100:00100000:0.0:1767319376.605042:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2180 00000100:00100000:0.0:1767319376.605051:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985154048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319376.605081:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319376.605464:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985154048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 413us (577us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319376.605473:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2180 00000100:00100000:3.0:1767319376.606922:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985154048:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319376.612455:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985154176:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319376.612510:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319376.612588:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154176 00000100:00100000:2.0:1767319376.612595:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154176 00000100:00100000:2.0:1767319376.612637:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2181 00000100:00100000:2.0:1767319376.612642:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985154176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319376.612666:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319376.612703:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985154176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (192us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319376.612708:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2181 00000100:00100000:1.0:1767319376.612818:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985154176:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319376.731670:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319377.757689:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319378.781240:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319379.803645:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319380.142629:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319380.142684:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920557440 00000100:00100000:1.0:1767319380.142694:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920557440 00000100:00100000:3.0:1767319380.142704:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319380.144567:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920557312 00000100:00100000:1.0:1767319380.144578:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920557312 00000100:00100000:1.0:1767319380.144611:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4583 00000100:00100000:1.0:1767319380.144622:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920557440:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319380.144693:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920557440:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (2072us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319380.144701:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4583 00000100:00100000:0.0:1767319380.144710:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5047 00000100:00100000:0.0:1767319380.144718:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920557312:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319380.144736:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319380.144772:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920557312:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (2069us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319380.144779:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5047 00000100:00080000:2.0:1767319380.147075:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319380.187322:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319380.187334:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319380.187338:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319380.187343:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319380.187347:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319380.187357:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4623 00000100:00080000:0.0:1767319380.187361:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319380.187368:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319380.187407:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633480 x1853163985154304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319380.187425:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319380.187428:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319380.187431:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319380.187434:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4623 00000100:00080000:0.0:1767319380.187437:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319380.187443:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319380.187444:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985154304:0@lo:38:kworker.0 00000100:00100000:0.0:1767319380.187459:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632a00 x1853163985154432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319380.187502:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319380.187507:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319380.187511:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319380.187512:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319380.187519:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4623 00000100:00080000:0.0:1767319380.187522:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319380.187527:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319380.187544:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630380 x1853163985154688/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319380.187584:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319380.187587:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319380.187589:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319380.187592:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4623 00000100:00080000:0.0:1767319380.187595:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319380.187598:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319380.187609:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985154944/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319380.187634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985154432:0@lo:38:kworker.0 00000100:00100000:3.0:1767319380.187663:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319380.187692:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985154688:0@lo:38:kworker.0 00000100:00100000:0.0:1767319380.187695:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985155072:0@lo:400:kworker.0 00000100:00100000:3.0:1767319380.187719:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319380.187741:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319380.187747:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985154944:0@lo:38:kworker.0 00000100:00100000:0.0:1767319380.187782:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319380.187788:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319380.187792:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985154816:0@lo:400:kworker.0 00000100:00100000:1.0:1767319380.187806:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985154560:0@lo:400:kworker.0 00000100:00100000:0.0:1767319380.187825:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319380.187847:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319380.187861:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155072 00000100:00100000:0.0:1767319380.187868:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155072 00000100:00100000:2.0:1767319380.187873:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985155200:0@lo:400:kworker.0 00000100:00100000:2.0:1767319380.187906:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319380.187933:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319380.188740:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155200 00000100:00100000:0.0:1767319380.188747:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155200 00000100:00100000:0.0:1767319380.188799:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3015 00000100:00100000:0.0:1767319380.188809:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985155072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319380.188868:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985155072:12345-0@lo:400:kworker.0 Request processed in 60us (1127us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319380.188875:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3015 00000100:00100000:0.0:1767319380.188893:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3016 00000100:00100000:0.0:1767319380.189281:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985155200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319380.189318:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154304 00000100:00100000:0.0:1767319380.189321:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985155200:12345-0@lo:400:kworker.0 Request processed in 421us (1413us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319380.189325:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154304 00000100:00100000:0.0:1767319380.189330:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3016 00000100:00100000:3.0:1767319380.189358:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154688 00000100:00100000:3.0:1767319380.189362:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154688 00000100:00100000:0.0:1767319380.189372:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154944 00000100:00100000:0.0:1767319380.189377:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154944 00000100:00100000:3.0:1767319380.189383:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154816 00000100:00100000:3.0:1767319380.189388:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154816 00000100:00100000:0.0:1767319380.189399:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5048 00000100:00100000:0.0:1767319380.189406:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985154304:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319380.189415:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4584 00000100:00100000:3.0:1767319380.189424:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985154816:12345-0@lo:400:kworker.0 00010000:00020000:0.0:1767319380.189426:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319380.189473:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985154304:12345-0@lo:38:kworker.0 Request processed in 66us (1963us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319380.189477:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985154816:12345-0@lo:400:kworker.0 Request processed in 53us (1651us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319380.189486:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5048 00000100:00100000:3.0:1767319380.189492:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4584 00000100:00100000:0.0:1767319380.189504:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5049 00000100:00100000:0.0:1767319380.189510:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985154688:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319380.189513:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5050 00000100:00100000:3.0:1767319380.189522:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985154944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319380.189525:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767319380.189535:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319380.189554:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985154688:12345-0@lo:38:kworker.0 Request processed in 42us (1834us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319380.189648:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5049 00000100:00100000:0.0:1767319380.189685:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985154816:0@lo:400:kworker.0 00000100:00100000:0.0:1767319380.189696:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985155072:0@lo:400:kworker.0 00000100:00100000:0.0:1767319380.189722:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985155200:0@lo:400:kworker.0 00000100:00080000:0.0:1767319380.189734:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319380.189737:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319380.189780:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154432 00000100:00100000:0.0:1767319380.189783:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154432 00000100:00100000:0.0:1767319380.189804:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1752 00000100:00100000:0.0:1767319380.189808:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985154432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319380.189818:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319380.189843:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985154432:12345-0@lo:38:kworker.0 Request processed in 34us (2181us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319380.189849:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1752 00000100:00100000:3.0:1767319380.189933:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985154944:12345-0@lo:38:kworker.0 Request processed in 45us (1781us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319380.189942:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5050 00000100:00080000:3.0:1767319380.189975:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319380.189979:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319380.189983:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985154944:0@lo:38:kworker.0 00000100:00080000:3.0:1767319380.189991:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319380.189995:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319380.189999:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985154432:0@lo:38:kworker.0 00000100:00080000:3.0:1767319380.190007:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319380.190011:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319380.190015:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985154688:0@lo:38:kworker.0 00000100:00080000:3.0:1767319380.190024:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319380.190027:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319380.190030:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985154304:0@lo:38:kworker.0 00000100:00100000:3.0:1767319380.196006:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985154560 00000100:00100000:3.0:1767319380.196022:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985154560 00000100:00080000:0.0:1767319380.196086:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319380.196129:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4585 00000100:00100000:0.0:1767319380.196140:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985154560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319380.196263:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985154560:12345-0@lo:400:kworker.0 Request processed in 125us (8416us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319380.196271:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4585 00000100:00100000:1.0:1767319380.196455:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985154560:0@lo:400:kworker.0 00010000:00010000:1.0:1767319380.827527:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319381.723382:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985155328:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319381.723442:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319381.723524:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155328 00000100:00100000:3.0:1767319381.723533:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155328 00000100:00100000:3.0:1767319381.723588:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2182 00000100:00100000:3.0:1767319381.723596:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985155328:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319381.723624:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319381.723646:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319381.723660:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985155456:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319381.723692:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319381.723717:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155456 00000100:00100000:2.0:1767319381.723721:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155456 00000100:00100000:2.0:1767319381.723757:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2183 00000100:00100000:2.0:1767319381.723762:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985155456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319381.723775:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319381.723801:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985155456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 39us (110us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319381.723806:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2183 00000100:00100000:0.0:1767319381.723840:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985155456:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319381.724381:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985155328:12345-0@lo:13:osp-pre-0-1.0 Request processed in 786us (936us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319381.724387:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2182 00000100:00100000:1.0:1767319381.724446:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985155328:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319381.851485:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319382.878154:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319383.900778:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319384.923519:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319385.258590:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319385.258707:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319385.258848:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920557696 00000100:00100000:3.0:1767319385.258859:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920557696 00000100:00100000:3.0:1767319385.258932:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920557568 00000100:00100000:3.0:1767319385.258936:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920557568 00000100:00100000:3.0:1767319385.258955:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4586 00000100:00100000:3.0:1767319385.258965:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920557696:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767319385.259063:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319385.259094:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920557696:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 129us (511us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319385.259103:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4586 00000100:00100000:3.0:1767319385.259125:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5051 00000100:00100000:3.0:1767319385.259130:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920557568:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319385.259151:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319385.259215:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920557568:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 84us (507us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319385.259222:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5051 00000100:00080000:2.0:1767319385.307688:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.307699:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319385.307702:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.307705:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.307710:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.307717:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.307721:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.307728:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319385.307767:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1c00 x1853163985155584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319385.307797:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.307801:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.307803:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.307806:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.307809:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.307812:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319385.307827:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3b80 x1853163985155712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319385.307881:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.307885:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.307887:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.307892:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.307894:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.307897:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319385.307909:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0380 x1853163985155968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319385.307939:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.307942:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.307945:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.307948:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.307950:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.307953:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319385.307964:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939500 x1853163985156224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319385.308001:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985156096:0@lo:400:kworker.0 00000100:00100000:2.0:1767319385.308061:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319385.308116:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319385.308122:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985156352:0@lo:400:kworker.0 00000100:00100000:2.0:1767319385.308147:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.308282:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985155840:0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.308312:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.308346:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319385.308351:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985156480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.308373:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.308402:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985156096 00000100:00100000:0.0:1767319385.308409:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985156096 00000100:00100000:0.0:1767319385.308471:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155840 00000100:00100000:0.0:1767319385.308474:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155840 00000100:00100000:0.0:1767319385.308500:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4587 00000100:00100000:0.0:1767319385.308508:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985156096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.308562:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985156096:12345-0@lo:400:kworker.0 Request processed in 53us (499us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319385.308570:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4587 00000100:00100000:0.0:1767319385.308588:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4588 00000100:00100000:0.0:1767319385.308595:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985155840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.308627:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985155840:12345-0@lo:400:kworker.0 Request processed in 32us (313us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319385.308633:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4588 00000100:00100000:0.0:1767319385.308652:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985156352 00000100:00100000:0.0:1767319385.308655:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985156352 00000100:00100000:0.0:1767319385.308690:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985156480 00000100:00100000:0.0:1767319385.308693:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985156480 00000100:00100000:0.0:1767319385.308710:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3017 00000100:00100000:0.0:1767319385.308716:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985156352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.308745:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985156352:12345-0@lo:400:kworker.0 Request processed in 29us (597us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319385.308750:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3017 00000100:00100000:0.0:1767319385.308761:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3018 00000100:00100000:0.0:1767319385.308767:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985156480:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.308791:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985156480:12345-0@lo:400:kworker.0 Request processed in 25us (417us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319385.308797:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3018 00000100:00100000:0.0:1767319385.308858:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985156480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.308869:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985155840:0@lo:400:kworker.0 00000100:00080000:1.0:1767319385.308935:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319385.308940:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319385.308942:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319385.313107:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985156352:0@lo:400:kworker.0 00000100:00100000:2.0:1767319385.313128:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985156096:0@lo:400:kworker.0 00000100:00100000:3.0:1767319385.313270:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985155584:0@lo:38:kworker.0 00000100:00100000:3.0:1767319385.313335:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319385.313386:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985155712:0@lo:38:kworker.0 00000100:00100000:3.0:1767319385.313414:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319385.313449:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985155968:0@lo:38:kworker.0 00000100:00100000:3.0:1767319385.313471:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319385.313501:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985156224:0@lo:38:kworker.0 00000100:00100000:3.0:1767319385.313521:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319385.313559:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155584 00000100:00100000:3.0:1767319385.313565:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155584 00000100:00100000:3.0:1767319385.313596:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155968 00000100:00100000:3.0:1767319385.313599:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155968 00000100:00100000:3.0:1767319385.313611:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985156224 00000100:00100000:3.0:1767319385.313613:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985156224 00000100:00100000:3.0:1767319385.313629:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5052 00000100:00100000:3.0:1767319385.313636:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985155584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319385.313656:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319385.313704:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985155584:12345-0@lo:38:kworker.0 Request processed in 66us (368us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319385.313710:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5052 00000100:00100000:3.0:1767319385.313727:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5053 00000100:00100000:3.0:1767319385.313731:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985155968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319385.313740:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319385.313764:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985155968:12345-0@lo:38:kworker.0 Request processed in 31us (292us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319385.313769:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5053 00000100:00100000:3.0:1767319385.313780:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5054 00000100:00100000:3.0:1767319385.313784:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985156224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319385.313793:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319385.313814:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985156224:12345-0@lo:38:kworker.0 Request processed in 29us (293us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319385.313820:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5054 00000100:00080000:3.0:1767319385.313868:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319385.313874:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319385.313877:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985155584:0@lo:38:kworker.0 00000100:00080000:3.0:1767319385.313886:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319385.313889:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319385.313892:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985155968:0@lo:38:kworker.0 00000100:00080000:3.0:1767319385.313899:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319385.313902:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319385.313905:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985156224:0@lo:38:kworker.0 00000100:00100000:0.0:1767319385.314159:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985155712 00000100:00100000:0.0:1767319385.314187:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985155712 00000100:00100000:0.0:1767319385.314221:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1753 00000100:00100000:0.0:1767319385.314226:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985155712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319385.314236:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319385.314332:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985155712:12345-0@lo:38:kworker.0 Request processed in 104us (916us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319385.314339:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1753 00000100:00080000:3.0:1767319385.314439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319385.314444:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319385.314446:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985155712:0@lo:38:kworker.0 00010000:00010000:0.0:1767319385.947758:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319386.844465:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985156736:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319386.844540:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319386.844643:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985156736 00000100:00100000:0.0:1767319386.844652:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985156736 00000100:00100000:0.0:1767319386.844714:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2184 00000100:00100000:0.0:1767319386.844723:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985156736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319386.844751:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319386.844825:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985156608:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319386.844852:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319386.844885:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985156608 00000100:00100000:1.0:1767319386.844889:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985156608 00000100:00100000:1.0:1767319386.844910:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2185 00000100:00100000:1.0:1767319386.844916:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985156608:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319386.844930:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319386.844959:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985156608:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (107us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319386.844967:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2185 00000100:00100000:1.0:1767319386.845016:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985156608:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319386.845544:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985156736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 821us (1002us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319386.845551:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2184 00000100:00100000:3.0:1767319386.846122:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985156736:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319386.971433:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319387.995675:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319389.020158:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767319389.357999:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920557824 offset 0 length 520: 4 00000100:00100000:1.0:1767319389.358047:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319389.358327:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558080 00000100:00100000:0.0:1767319389.358337:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558080 00000100:00100000:0.0:1767319389.358453:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4589 00000100:00100000:0.0:1767319389.358465:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920558080:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319389.358557:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920558080:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 95us (510us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319389.358565:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4589 00000100:00100000:3.0:1767319389.360321:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319389.360406:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920557952 00000100:00100000:2.0:1767319389.360415:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920557952 00000100:00100000:2.0:1767319389.360462:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5055 00000100:00100000:2.0:1767319389.360479:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920557952:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319389.360506:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319389.360573:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920557952:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (257us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319389.360581:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5055 00000100:00080000:1.0:1767319389.361787:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319389.403391:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319364/real 1767319364] req@ffff8ce1b34d2300 x1853163985150336/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319389 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319389.403422:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b34d2300 x1853163985150336/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319389 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319389.403435:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.403503:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319389.403507:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985150336:0@lo:250:kworker.0 00000100:00080000:1.0:1767319389.407075:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319389.407086:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319389.407089:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319389.407097:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4613 00000100:00080000:1.0:1767319389.407101:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319389.407108:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319389.407145:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c000 x1853163985156864/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319389.407200:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319389.407205:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319389.407206:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319389.407210:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4633 00000100:00080000:1.0:1767319389.407212:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319389.407215:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319389.407231:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f480 x1853163985156992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319389.407239:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319389.407242:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319389.407244:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319389.407248:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4633 00000100:00080000:1.0:1767319389.407250:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319389.407253:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319389.407266:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73dc00 x1853163985157120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319389.407314:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319389.407318:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319389.407320:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319389.407323:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4633 00000100:00080000:1.0:1767319389.407326:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319389.407329:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319389.407339:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c380 x1853163985157376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319389.407367:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319389.407370:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319389.407371:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319389.407374:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4633 00000100:00080000:1.0:1767319389.407377:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319389.407380:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319389.407390:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e680 x1853163985157632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319389.407451:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985157760:0@lo:400:kworker.0 00000100:00100000:1.0:1767319389.407517:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319389.407581:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319389.407598:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985157504:0@lo:400:kworker.0 00000100:00100000:3.0:1767319389.407607:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985157888:0@lo:400:kworker.0 00000100:00100000:1.0:1767319389.407645:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319389.407646:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319389.407682:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319389.407687:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985157248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319389.407697:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985157760 00000100:00100000:3.0:1767319389.407703:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319389.407705:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985157760 00000100:00100000:3.0:1767319389.407730:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985157888 00000100:00100000:3.0:1767319389.407734:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985157888 00000100:00100000:2.0:1767319389.407828:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985157504 00000100:00100000:2.0:1767319389.407834:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985157504 00000100:00100000:2.0:1767319389.407877:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985157248 00000100:00100000:2.0:1767319389.407881:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985157248 00000100:00100000:2.0:1767319389.407903:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4590 00000100:00100000:2.0:1767319389.407922:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985157504:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319389.407974:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985157504:12345-0@lo:400:kworker.0 Request processed in 54us (331us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319389.407981:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4590 00000100:00100000:2.0:1767319389.407999:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4591 00000100:00100000:2.0:1767319389.408005:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985157248:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319389.408037:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985157248:12345-0@lo:400:kworker.0 Request processed in 33us (333us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319389.408043:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4591 00000100:00100000:2.0:1767319389.408082:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985157248:0@lo:400:kworker.0 00000100:00100000:2.0:1767319389.408106:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985157504:0@lo:400:kworker.0 00000100:00080000:2.0:1767319389.408148:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319389.408153:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319389.408155:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319389.408242:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3019 00000100:00100000:1.0:1767319389.408251:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a05450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985157760:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319389.408285:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a05450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985157760:12345-0@lo:400:kworker.0 Request processed in 37us (768us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319389.408290:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3019 00000100:00100000:1.0:1767319389.408557:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3020 00000100:00100000:1.0:1767319389.408564:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985157888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319389.408608:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985157888:12345-0@lo:400:kworker.0 Request processed in 44us (964us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319389.408614:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3020 00000100:00100000:2.0:1767319389.408676:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985157888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319389.408700:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985157760:0@lo:400:kworker.0 00000100:00100000:0.0:1767319389.409272:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985156864:0@lo:250:kworker.0 00000400:00000100:0.0:1767319389.409301:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985156864 offset 0 length 520: 4 00000100:00100000:0.0:1767319389.409321:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985156992:0@lo:38:kworker.0 00000100:00100000:0.0:1767319389.409340:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319389.409359:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985157120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319389.409373:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319389.409385:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985157376:0@lo:38:kworker.0 00000100:00100000:0.0:1767319389.409396:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319389.409415:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985157632:0@lo:38:kworker.0 00000100:00100000:0.0:1767319389.409425:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319389.409448:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985157120 00000100:00100000:0.0:1767319389.409452:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985157120 00000100:00100000:0.0:1767319389.409470:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1754 00000100:00100000:0.0:1767319389.409474:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985157120:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319389.409488:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.409510:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985157120:12345-0@lo:38:kworker.0 Request processed in 35us (135us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319389.409514:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1754 00000100:00100000:0.0:1767319389.409533:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985156992 00000100:00100000:0.0:1767319389.409536:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985156992 00000100:00100000:0.0:1767319389.409553:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985157376 00000100:00100000:0.0:1767319389.409556:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985157376 00000100:00100000:0.0:1767319389.409567:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985157632 00000100:00100000:0.0:1767319389.409569:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985157632 00000100:00100000:0.0:1767319389.409578:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5056 00000100:00100000:0.0:1767319389.409582:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985156992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319389.409603:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.409628:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985156992:12345-0@lo:38:kworker.0 Request processed in 45us (288us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319389.409634:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5056 00000100:00100000:0.0:1767319389.409641:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5057 00000100:00100000:0.0:1767319389.409644:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985157376:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319389.409650:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.409669:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985157376:12345-0@lo:38:kworker.0 Request processed in 23us (271us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319389.409673:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5057 00000100:00100000:0.0:1767319389.409683:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5058 00000100:00100000:0.0:1767319389.409686:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985157632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319389.409694:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.409713:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985157632:12345-0@lo:38:kworker.0 Request processed in 25us (286us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319389.409719:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5058 00000100:00080000:0.0:1767319389.409750:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.409755:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.409758:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985156992:0@lo:38:kworker.0 00000100:00080000:0.0:1767319389.409766:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.409768:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.409771:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985157120:0@lo:38:kworker.0 00000100:00080000:0.0:1767319389.409777:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.409779:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.409782:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985157376:0@lo:38:kworker.0 00000100:00080000:0.0:1767319389.409788:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.409791:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.409793:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985157632:0@lo:38:kworker.0 00010000:00010000:2.0:1767319390.044309:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319391.071080:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319391.963635:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985158016:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319391.963713:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319391.963770:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319391.963779:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985158144:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319391.963802:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319391.963969:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158016 00000100:00100000:0.0:1767319391.963978:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158016 00000100:00100000:0.0:1767319391.964046:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158144 00000100:00100000:0.0:1767319391.964050:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158144 00000100:00100000:0.0:1767319391.964070:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2186 00000100:00100000:0.0:1767319391.964078:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985158016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319391.964107:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319391.964153:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2187 00000100:00100000:0.0:1767319391.964154:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985158016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (440us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319391.964189:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2186 00000100:00100000:3.0:1767319391.964190:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985158144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319391.964221:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319391.964266:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985158144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (462us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319391.964272:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2187 00000100:00100000:0.0:1767319391.964501:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985158144:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319391.964569:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319391.964574:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319391.964594:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985158016:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319392.097157:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319393.115570:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319394.139717:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319394.485605:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319394.485660:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319394.485751:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558336 00000100:00100000:1.0:1767319394.485762:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558336 00000100:00100000:1.0:1767319394.485832:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558208 00000100:00100000:1.0:1767319394.485836:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558208 00000100:00080000:3.0:1767319394.485836:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319394.485873:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4592 00000100:00100000:0.0:1767319394.485881:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5059 00000100:00100000:1.0:1767319394.485886:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920558336:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319394.485893:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920558208:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319394.485911:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319394.486068:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920558208:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 176us (405us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319394.486075:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5059 00000100:00100000:1.0:1767319394.493616:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920558336:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 7720us (8006us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319394.493632:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4592 00000100:00080000:1.0:1767319394.523289:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319394.523299:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319394.523303:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319394.523307:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319394.523312:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319394.523320:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4637 00000100:00080000:1.0:1767319394.523326:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319394.523333:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319394.523371:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163985158272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319394.523393:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319394.523397:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319394.523399:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319394.523403:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4637 00000100:00080000:1.0:1767319394.523405:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319394.523409:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319394.523421:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f800 x1853163985158400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319394.523463:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319394.523467:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319394.523469:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319394.523473:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4637 00000100:00080000:1.0:1767319394.523476:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767319394.523477:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985158528:0@lo:400:kworker.0 00000100:00080000:1.0:1767319394.523482:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319394.523508:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f100 x1853163985158656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319394.523544:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319394.523547:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319394.523549:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319394.523552:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4637 00000100:00080000:1.0:1767319394.523555:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319394.523558:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319394.523564:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319394.523575:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d880 x1853163985158912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319394.523614:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319394.523616:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985158784:0@lo:400:kworker.0 00000100:00100000:2.0:1767319394.523622:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985159168:0@lo:400:kworker.0 00000100:00100000:3.0:1767319394.523709:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985159040:0@lo:400:kworker.0 00000100:00100000:1.0:1767319394.523743:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319394.523743:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319394.523791:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158528 00000100:00100000:1.0:1767319394.523794:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158784 00000100:00100000:2.0:1767319394.523797:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158528 00000100:00100000:1.0:1767319394.523801:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158784 00000100:00100000:0.0:1767319394.523867:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985158272:0@lo:38:kworker.0 00000100:00100000:0.0:1767319394.523888:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319394.523904:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985158400:0@lo:38:kworker.0 00000100:00100000:0.0:1767319394.523917:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319394.523929:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985158656:0@lo:38:kworker.0 00000100:00100000:0.0:1767319394.523947:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319394.523963:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985158912:0@lo:38:kworker.0 00000100:00100000:0.0:1767319394.523984:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319394.524010:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159168 00000100:00100000:0.0:1767319394.524014:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159168 00000100:00100000:2.0:1767319394.524063:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158272 00000100:00100000:2.0:1767319394.524067:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158272 00000100:00100000:2.0:1767319394.524077:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158656 00000100:00100000:2.0:1767319394.524090:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158656 00000100:00100000:2.0:1767319394.524098:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158912 00000100:00100000:2.0:1767319394.524101:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158912 00000100:00100000:2.0:1767319394.524111:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4593 00000100:00100000:2.0:1767319394.524119:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985158528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319394.524189:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985158528:12345-0@lo:400:kworker.0 Request processed in 70us (627us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319394.524200:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4593 00000100:00100000:2.0:1767319394.524217:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5060 00000100:00100000:2.0:1767319394.524221:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985158272:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319394.524238:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319394.524268:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985158272:12345-0@lo:38:kworker.0 Request processed in 45us (378us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319394.524274:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5060 00000100:00100000:2.0:1767319394.524284:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5061 00000100:00100000:2.0:1767319394.524288:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985158656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319394.524297:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319394.524318:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985158656:12345-0@lo:38:kworker.0 Request processed in 28us (370us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319394.524323:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5061 00000100:00100000:2.0:1767319394.524334:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5062 00000100:00100000:2.0:1767319394.524339:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985158912:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319394.524347:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319394.524363:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985158912:12345-0@lo:38:kworker.0 Request processed in 24us (379us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319394.524366:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5062 00000100:00100000:3.0:1767319394.525471:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319394.525529:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159040 00000100:00100000:3.0:1767319394.525536:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159040 00000100:00100000:1.0:1767319394.530329:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4594 00000100:00100000:0.0:1767319394.530341:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3021 00000100:00100000:1.0:1767319394.530347:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985158784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319394.530353:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985159168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319394.530488:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985158784:12345-0@lo:400:kworker.0 Request processed in 148us (6750us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319394.530506:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4594 00000100:00080000:2.0:1767319394.530562:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319394.530564:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985158784:0@lo:400:kworker.0 00000100:00080000:2.0:1767319394.530567:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319394.530572:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319394.530601:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985158528:0@lo:400:kworker.0 00000100:00100000:2.0:1767319394.530703:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985158400 00000100:00100000:2.0:1767319394.530710:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985158400 00000100:00080000:1.0:1767319394.530720:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767319394.530732:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1755 00000100:00100000:2.0:1767319394.530736:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985158400:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319394.530752:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319394.530775:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985158400:12345-0@lo:38:kworker.0 Request processed in 39us (6857us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319394.530880:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1755 00000100:00080000:1.0:1767319394.531407:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.531413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985158272:0@lo:38:kworker.0 00000100:00100000:0.0:1767319394.531423:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985159168:12345-0@lo:400:kworker.0 Request processed in 1072us (7683us total) trans 0 rc 0/0 00000100:00080000:1.0:1767319394.531430:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319394.531434:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.531438:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985158400:0@lo:38:kworker.0 00000100:00080000:1.0:1767319394.531448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319394.531451:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.531454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985158656:0@lo:38:kworker.0 00000100:00080000:1.0:1767319394.531463:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319394.531466:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.531469:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985158912:0@lo:38:kworker.0 00000100:00100000:0.0:1767319394.531586:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3021 00000100:00100000:3.0:1767319394.531923:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3022 00000100:00100000:3.0:1767319394.531934:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985159040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319394.531990:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985159040:12345-0@lo:400:kworker.0 Request processed in 59us (6523us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319394.531996:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3022 00000100:00100000:1.0:1767319394.532149:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985159040:0@lo:400:kworker.0 00000100:00100000:2.0:1767319394.532402:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985159168:0@lo:400:kworker.0 00010000:00010000:1.0:1767319395.163679:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319396.190534:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319397.083401:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319397.085493:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319397.085518:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985159424:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319397.085585:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319397.085634:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159424 00000100:00100000:1.0:1767319397.085641:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159424 00000100:00100000:1.0:1767319397.085696:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2188 00000100:00100000:1.0:1767319397.085704:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985159424:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319397.085734:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319397.085777:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985159424:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (192us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319397.085785:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2188 00000100:00100000:1.0:1767319397.085831:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985159424:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319397.085867:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319397.086493:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985159296:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319397.086525:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319397.086910:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159296 00000100:00100000:1.0:1767319397.086915:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159296 00000100:00100000:1.0:1767319397.086996:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2189 00000100:00100000:1.0:1767319397.087003:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985159296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319397.087022:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319397.087063:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985159296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (537us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319397.087070:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2189 00000100:00100000:3.0:1767319397.087145:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985159296:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319397.087304:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767319397.211469:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319398.235417:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319399.259336:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319399.595511:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319399.595615:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319399.603453:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558592 00000100:00100000:2.0:1767319399.603469:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558592 00000100:00100000:2.0:1767319399.603560:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558464 00000100:00100000:2.0:1767319399.603565:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558464 00000100:00100000:2.0:1767319399.603590:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4595 00000100:00100000:2.0:1767319399.603603:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920558592:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767319399.603915:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319399.603951:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5063 00000100:00100000:1.0:1767319399.603957:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920558464:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319399.603977:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319399.604030:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920558592:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 431us (8528us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319399.604038:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4595 00000100:00100000:1.0:1767319399.616287:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920558464:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 12313us (20657us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319399.616302:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5063 00000100:00080000:2.0:1767319399.643457:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319399.643468:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319399.643472:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319399.643476:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319399.643480:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319399.643488:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4642 00000100:00080000:2.0:1767319399.643493:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319399.643501:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319399.643555:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938380 x1853163985159552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319399.643588:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319399.643591:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319399.643593:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319399.643597:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4642 00000100:00080000:2.0:1767319399.643599:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319399.643603:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319399.643617:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985159680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319399.643677:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319399.643681:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319399.643683:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319399.643687:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4642 00000100:00080000:2.0:1767319399.643689:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319399.643693:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319399.643705:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938700 x1853163985159936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319399.643752:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319399.643756:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319399.643757:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319399.643761:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4642 00000100:00080000:2.0:1767319399.643764:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319399.643767:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319399.643779:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a300 x1853163985160192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319399.643821:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985160320:0@lo:400:kworker.0 00000100:00100000:2.0:1767319399.643894:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319399.644039:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985160448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319399.644082:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319399.644366:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319399.644374:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985160064:0@lo:400:kworker.0 00000100:00100000:2.0:1767319399.644412:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319399.644485:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985159808:0@lo:400:kworker.0 00000100:00100000:3.0:1767319399.644515:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319399.644590:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985159552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319399.644623:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319399.648520:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160320 00000100:00100000:0.0:1767319399.648535:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160320 00000100:00100000:0.0:1767319399.648610:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160448 00000100:00100000:0.0:1767319399.648613:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160448 00000100:00100000:0.0:1767319399.648645:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3023 00000100:00100000:0.0:1767319399.648653:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985160320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319399.649516:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985160320:12345-0@lo:400:kworker.0 Request processed in 860us (5618us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319399.649526:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3023 00000100:00100000:0.0:1767319399.649550:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3024 00000100:00100000:0.0:1767319399.649558:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985160448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319399.649589:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985160448:12345-0@lo:400:kworker.0 Request processed in 33us (5507us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319399.649594:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3024 00000100:00080000:0.0:1767319399.649620:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319399.649624:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319399.649701:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160064 00000100:00100000:3.0:1767319399.649707:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160064 00000100:00100000:2.0:1767319399.650268:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985160320:0@lo:400:kworker.0 00000100:00080000:2.0:1767319399.650338:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319399.650348:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159808 00000100:00100000:2.0:1767319399.650351:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159808 00000100:00100000:2.0:1767319399.650385:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159552 00000100:00100000:2.0:1767319399.650388:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159552 00000100:00100000:2.0:1767319399.650415:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4596 00000100:00100000:2.0:1767319399.650422:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985159808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319399.650465:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985159808:12345-0@lo:400:kworker.0 Request processed in 44us (5950us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319399.650471:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4596 00000100:00100000:2.0:1767319399.650483:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5064 00000100:00100000:2.0:1767319399.650486:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985159552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319399.650504:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319399.650529:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985159552:12345-0@lo:38:kworker.0 Request processed in 42us (5906us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319399.650535:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5064 00000100:00080000:2.0:1767319399.650549:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319399.650641:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4597 00000100:00100000:3.0:1767319399.650649:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985160064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319399.650695:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985160064:12345-0@lo:400:kworker.0 Request processed in 36us (6271us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319399.650700:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4597 00000100:00100000:3.0:1767319399.650733:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985160064:0@lo:400:kworker.0 00000100:00100000:2.0:1767319399.657714:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985159808:0@lo:400:kworker.0 00000100:00100000:2.0:1767319399.658546:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985160448:0@lo:400:kworker.0 00000100:00080000:1.0:1767319399.658643:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319399.658650:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319399.658654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985159552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319399.658664:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985159680:0@lo:38:kworker.0 00000100:00100000:1.0:1767319399.658921:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319399.659973:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985159936:0@lo:38:kworker.0 00000100:00100000:1.0:1767319399.660012:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319399.660040:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985160192:0@lo:38:kworker.0 00000100:00100000:1.0:1767319399.660060:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319399.660110:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159936 00000100:00100000:3.0:1767319399.660120:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159936 00000100:00100000:3.0:1767319399.660184:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160192 00000100:00100000:3.0:1767319399.660188:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160192 00000100:00100000:3.0:1767319399.660213:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5065 00000100:00100000:3.0:1767319399.660220:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985159936:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319399.660241:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319399.660286:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985159936:12345-0@lo:38:kworker.0 Request processed in 64us (272us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319399.660292:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5065 00000100:00100000:3.0:1767319399.660308:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5066 00000100:00100000:3.0:1767319399.660312:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985160192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319399.660320:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319399.660343:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985160192:12345-0@lo:38:kworker.0 Request processed in 30us (282us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319399.660348:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5066 00000100:00080000:1.0:1767319399.660567:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319399.660573:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319399.660576:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985159936:0@lo:38:kworker.0 00000100:00080000:1.0:1767319399.660585:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319399.660588:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319399.660591:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985160192:0@lo:38:kworker.0 00000100:00100000:0.0:1767319399.661224:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985159680 00000100:00100000:0.0:1767319399.661231:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985159680 00000100:00100000:0.0:1767319399.661264:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1756 00000100:00100000:0.0:1767319399.661269:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985159680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319399.661282:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319399.661334:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985159680:12345-0@lo:38:kworker.0 Request processed in 63us (2412us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319399.661341:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1756 00000100:00080000:1.0:1767319399.661461:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319399.661466:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319399.661470:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985159680:0@lo:38:kworker.0 00010000:00010000:2.0:1767319400.286887:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319401.307575:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319402.204710:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985160576:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319402.204784:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319402.204896:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160576 00000100:00100000:1.0:1767319402.204905:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160576 00000100:00100000:1.0:1767319402.204982:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2190 00000100:00100000:1.0:1767319402.204992:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985160576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319402.205023:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319402.205087:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985160576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (286us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319402.205095:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2190 00000100:00100000:1.0:1767319402.205146:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985160576:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319402.205534:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319402.210487:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985160704:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319402.210563:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319402.211034:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160704 00000100:00100000:1.0:1767319402.211043:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160704 00000100:00100000:1.0:1767319402.211129:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2191 00000100:00100000:1.0:1767319402.211142:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985160704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319402.211210:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319402.211259:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985160704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 122us (696us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319402.211268:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2191 00000100:00080000:2.0:1767319402.213007:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319402.213095:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985160704:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319402.331464:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319403.362843:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319404.380083:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319404.718284:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319404.718392:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319404.718507:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558848 00000100:00100000:2.0:1767319404.718516:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558848 00000100:00100000:2.0:1767319404.718592:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558720 00000100:00100000:2.0:1767319404.718595:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558720 00000100:00100000:2.0:1767319404.718614:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4598 00000100:00100000:2.0:1767319404.718628:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920558848:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319404.718703:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920558848:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (1771us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319404.718711:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4598 00000100:00100000:2.0:1767319404.718729:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5067 00000100:00100000:2.0:1767319404.718734:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920558720:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319404.718753:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319404.718774:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920558720:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (387us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319404.718783:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5067 00000100:00080000:3.0:1767319404.719401:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319404.763496:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319404.763507:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319404.763510:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319404.763513:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319404.763518:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319404.763526:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4647 00000100:00080000:3.0:1767319404.763530:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319404.763537:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319404.763588:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519c00 x1853163985160832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319404.763613:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319404.763617:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319404.763619:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319404.763622:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4647 00000100:00080000:3.0:1767319404.763624:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319404.763628:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319404.763641:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518e00 x1853163985160960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319404.763687:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319404.763692:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319404.763694:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319404.763697:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4647 00000100:00080000:3.0:1767319404.763699:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319404.763702:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319404.763713:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518700 x1853163985161216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319404.763742:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319404.763745:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319404.763747:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319404.763750:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4647 00000100:00080000:3.0:1767319404.763753:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319404.763755:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319404.763766:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985161344:0@lo:400:kworker.0 00000100:00100000:3.0:1767319404.763825:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319404.763897:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161344 00000100:00100000:3.0:1767319404.763904:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161344 00000100:00100000:3.0:1767319404.763952:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4599 00000100:00100000:3.0:1767319404.763960:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985161344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319404.764006:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985161344:12345-0@lo:400:kworker.0 Request processed in 47us (181us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319404.764014:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4599 00000100:00080000:3.0:1767319404.764035:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319404.764063:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985161344:0@lo:400:kworker.0 00000100:00100000:3.0:1767319404.764088:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985161472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319404.764110:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319404.764116:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985161600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319404.764142:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319404.764439:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985161088:0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.764468:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319404.764495:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319404.764500:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985161728:0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.764519:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319404.764543:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161600 00000100:00100000:0.0:1767319404.764546:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161600 00000100:00100000:0.0:1767319404.764588:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161728 00000100:00100000:0.0:1767319404.764590:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161728 00000100:00100000:0.0:1767319404.764607:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3025 00000100:00100000:0.0:1767319404.764612:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985161600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.764644:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985161600:12345-0@lo:400:kworker.0 Request processed in 33us (502us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319404.764650:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3025 00000100:00100000:0.0:1767319404.764662:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3026 00000100:00100000:0.0:1767319404.764668:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985161728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.764693:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985161728:12345-0@lo:400:kworker.0 Request processed in 26us (173us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319404.764698:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3026 00000100:00080000:0.0:1767319404.764718:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319404.764722:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319404.764738:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985161600:0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.764759:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985161728:0@lo:400:kworker.0 00000100:00100000:2.0:1767319404.764941:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985160832:0@lo:38:kworker.0 00000100:00100000:2.0:1767319404.764971:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319404.765003:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985160960:0@lo:38:kworker.0 00000100:00100000:2.0:1767319404.765026:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319404.765055:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985161216:0@lo:38:kworker.0 00000100:00100000:2.0:1767319404.765075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319404.765090:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985161472:0@lo:38:kworker.0 00000100:00100000:2.0:1767319404.765108:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319404.765138:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161088 00000100:00100000:2.0:1767319404.765141:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161088 00000100:00100000:2.0:1767319404.765188:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160832 00000100:00100000:2.0:1767319404.765191:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160832 00000100:00100000:2.0:1767319404.765205:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161216 00000100:00100000:2.0:1767319404.765207:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161216 00000100:00100000:1.0:1767319404.765332:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985160960 00000100:00100000:1.0:1767319404.765336:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985160960 00000100:00100000:1.0:1767319404.765359:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1757 00000100:00100000:1.0:1767319404.765364:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985160960:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319404.765382:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319404.765420:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985160960:12345-0@lo:38:kworker.0 Request processed in 55us (393us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319404.765425:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1757 00000100:00100000:1.0:1767319404.765443:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161472 00000100:00100000:1.0:1767319404.765446:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161472 00000100:00100000:1.0:1767319404.765458:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4600 00000100:00100000:1.0:1767319404.765464:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985161088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319404.765495:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985161088:12345-0@lo:400:kworker.0 Request processed in 32us (1027us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319404.765501:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4600 00000100:00100000:1.0:1767319404.765513:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5068 00000100:00100000:1.0:1767319404.765517:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985160832:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319404.765527:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319404.765549:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985160832:12345-0@lo:38:kworker.0 Request processed in 31us (578us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319404.765555:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5068 00000100:00100000:1.0:1767319404.765565:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5069 00000100:00100000:1.0:1767319404.765580:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985161216:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319404.765587:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319404.765603:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985161216:12345-0@lo:38:kworker.0 Request processed in 22us (529us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319404.765606:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5069 00000100:00100000:1.0:1767319404.765613:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5070 00000100:00100000:1.0:1767319404.765615:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985161472:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319404.765620:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319404.765631:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985161472:12345-0@lo:38:kworker.0 Request processed in 15us (524us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319404.765635:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5070 00000100:00080000:1.0:1767319404.765651:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319404.765655:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319404.765657:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985160832:0@lo:38:kworker.0 00000100:00080000:1.0:1767319404.765661:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319404.765663:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319404.765665:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985160960:0@lo:38:kworker.0 00000100:00080000:1.0:1767319404.765669:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319404.765671:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319404.765672:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985161216:0@lo:38:kworker.0 00000100:00080000:1.0:1767319404.765676:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319404.765677:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319404.765678:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985161472:0@lo:38:kworker.0 00000100:00080000:0.0:1767319404.765889:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319404.765907:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985161088:0@lo:400:kworker.0 00010000:00010000:1.0:1767319405.403551:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319406.431525:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319407.323903:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319407.323921:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985161856:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319407.323989:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319407.324071:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161856 00000100:00100000:0.0:1767319407.324080:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161856 00000100:00100000:0.0:1767319407.324141:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2192 00000100:00100000:0.0:1767319407.324151:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985161856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319407.324198:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319407.324252:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985161856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319407.324266:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2192 00000100:00080000:3.0:1767319407.324334:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319407.324371:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985161856:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319407.324874:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985161984:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319407.324909:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319407.324957:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985161984 00000100:00100000:1.0:1767319407.324962:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985161984 00000100:00100000:1.0:1767319407.325000:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2193 00000100:00100000:1.0:1767319407.325007:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985161984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319407.325023:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319407.325064:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985161984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (143us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319407.325070:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2193 00000100:00100000:1.0:1767319407.325101:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985161984:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319407.325320:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319407.451431:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319408.475412:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319409.499741:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319409.834031:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319409.834200:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920559104 00000100:00100000:0.0:1767319409.834243:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920559104 00000100:00100000:0.0:1767319409.834362:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4601 00000100:00100000:0.0:1767319409.834382:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920559104:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319409.834469:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920559104:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (448us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319409.834477:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4601 00000100:00080000:3.0:1767319409.836462:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319409.846967:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319409.847079:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920558976 00000100:00100000:2.0:1767319409.847089:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920558976 00000100:00100000:2.0:1767319409.847142:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5071 00000100:00100000:2.0:1767319409.847150:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920558976:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319409.847224:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319409.847298:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920558976:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 147us (341us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319409.847307:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5071 00000100:00080000:1.0:1767319409.883405:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.883413:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319409.883416:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.883419:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.883423:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.883429:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.883433:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319409.883439:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319409.883475:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524e300 x1853163985162112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319409.883487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.883490:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.883491:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.883714:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.883718:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319409.883721:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319409.883734:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985162240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319409.883781:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767319409.883935:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.883938:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.883940:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.883943:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.883945:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319409.883948:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319409.883958:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524dc00 x1853163985162496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319409.883985:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.883988:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.883989:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.883991:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.883993:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319409.883994:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319409.884003:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c380 x1853163985162752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319409.884051:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985162624:0@lo:400:kworker.0 00000100:00100000:0.0:1767319409.884108:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319409.884157:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319409.884190:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985162880:0@lo:400:kworker.0 00000100:00100000:0.0:1767319409.884225:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319409.884256:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985162624 00000100:00100000:0.0:1767319409.884262:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985162624 00000100:00100000:0.0:1767319409.884313:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4602 00000100:00100000:0.0:1767319409.884321:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985162624:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319409.884371:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985162624:12345-0@lo:400:kworker.0 Request processed in 50us (261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319409.884379:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4602 00000100:00100000:0.0:1767319409.884419:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985162624:0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.885545:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985162112:0@lo:38:kworker.0 00000100:00100000:2.0:1767319409.885589:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.885628:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985162240:0@lo:38:kworker.0 00000100:00100000:2.0:1767319409.885646:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.885663:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985162496:0@lo:38:kworker.0 00000100:00100000:2.0:1767319409.885676:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.885685:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985162752:0@lo:38:kworker.0 00000100:00100000:2.0:1767319409.885696:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.885723:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985162880 00000100:00100000:2.0:1767319409.885728:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985162880 00000100:00100000:2.0:1767319409.885756:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3027 00000100:00100000:2.0:1767319409.885762:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985162880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.885791:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985162880:12345-0@lo:400:kworker.0 Request processed in 31us (1567us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319409.885796:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3027 00000100:00100000:2.0:1767319409.885812:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985162240 00000100:00100000:2.0:1767319409.885814:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985162240 00000100:00100000:2.0:1767319409.885826:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1758 00000100:00100000:2.0:1767319409.885829:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985162240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319409.885843:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319409.885863:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985162240:12345-0@lo:38:kworker.0 Request processed in 33us (216us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319409.885867:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1758 00000100:00100000:2.0:1767319409.885877:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985162112 00000100:00100000:2.0:1767319409.885879:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985162112 00000100:00100000:2.0:1767319409.885889:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985162496 00000100:00100000:2.0:1767319409.885891:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985162496 00000100:00100000:2.0:1767319409.885899:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985162752 00000100:00100000:2.0:1767319409.885900:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985162752 00000100:00100000:2.0:1767319409.885906:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5072 00000100:00100000:2.0:1767319409.885908:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985162112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319409.885914:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319409.885928:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985162112:12345-0@lo:38:kworker.0 Request processed in 19us (339us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319409.885931:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5072 00000100:00100000:2.0:1767319409.885937:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5073 00000100:00100000:2.0:1767319409.885939:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985162496:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319409.885944:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319409.885953:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985162496:12345-0@lo:38:kworker.0 Request processed in 13us (277us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319409.885956:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5073 00000100:00100000:2.0:1767319409.885961:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5074 00000100:00100000:2.0:1767319409.885963:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985162752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319409.885968:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319409.885977:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985162752:12345-0@lo:38:kworker.0 Request processed in 13us (280us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319409.885980:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5074 00000100:00080000:2.0:1767319409.886004:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319409.886007:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319409.886009:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985162112:0@lo:38:kworker.0 00000100:00080000:2.0:1767319409.886014:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319409.886018:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319409.886020:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985162240:0@lo:38:kworker.0 00000100:00080000:2.0:1767319409.886023:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319409.886025:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319409.886027:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985162496:0@lo:38:kworker.0 00000100:00080000:2.0:1767319409.886031:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319409.886033:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319409.886034:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985162752:0@lo:38:kworker.0 00000100:00080000:2.0:1767319409.886068:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319409.886071:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319409.887469:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985162880:0@lo:400:kworker.0 00000100:00100000:3.0:1767319409.887932:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985163008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319409.887986:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319409.888014:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985162368:0@lo:400:kworker.0 00000100:00100000:3.0:1767319409.888030:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.888087:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163008 00000100:00100000:2.0:1767319409.888093:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163008 00000100:00100000:2.0:1767319409.888135:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3028 00000100:00100000:2.0:1767319409.888142:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985163008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.888222:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985163008:12345-0@lo:400:kworker.0 Request processed in 80us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319409.888229:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3028 00000100:00100000:2.0:1767319409.888264:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985162368 00000100:00100000:2.0:1767319409.888267:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985162368 00000100:00100000:2.0:1767319409.888294:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4603 00000100:00100000:2.0:1767319409.888299:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985162368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.888328:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985162368:12345-0@lo:400:kworker.0 Request processed in 29us (296us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319409.888335:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4603 00000100:00080000:3.0:1767319409.889335:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319409.889340:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319409.889375:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985163008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319409.889384:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985162368:0@lo:400:kworker.0 00010000:00010000:0.0:1767319410.527222:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319411.547488:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319412.443543:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985163136:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319412.443631:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319412.443678:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985163264:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319412.443711:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319412.443749:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163136 00000100:00100000:3.0:1767319412.443757:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163136 00000100:00080000:0.0:1767319412.443820:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319412.443829:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163264 00000100:00100000:3.0:1767319412.443833:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163264 00000100:00100000:0.0:1767319412.443853:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2194 00000100:00100000:3.0:1767319412.443860:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2195 00000100:00100000:0.0:1767319412.443866:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985163136:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319412.443881:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985163264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319412.443895:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:3.0:1767319412.443905:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319412.443943:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985163136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (312us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319412.443951:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2194 00000100:00100000:3.0:1767319412.443966:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985163264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (253us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319412.443973:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2195 00000100:00080000:0.0:1767319412.443984:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319412.444022:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985163264:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319412.445978:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985163136:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319412.572116:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319413.600957:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319414.619413:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319414.957913:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767319414.957965:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920559232 offset 0 length 520: 4 00000100:00100000:1.0:1767319414.957997:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319414.958147:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920559488 00000100:00100000:0.0:1767319414.958211:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920559488 00000100:00100000:0.0:1767319414.958307:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920559360 00000100:00100000:0.0:1767319414.958311:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920559360 00000100:00100000:0.0:1767319414.958334:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4604 00000100:00100000:0.0:1767319414.958347:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920559488:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319414.958450:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920559488:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 106us (554us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319414.958459:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4604 00000100:00100000:0.0:1767319414.958480:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5075 00000100:00100000:0.0:1767319414.958485:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920559360:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319414.958586:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319414.958622:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920559360:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 135us (623us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319414.958628:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5075 00000100:00080000:1.0:1767319414.958823:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319415.003386:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319415.003401:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319415.003405:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319415.003411:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319415.003416:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319415.003428:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4658 00000100:00080000:3.0:1767319415.003434:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319415.003443:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319415.003497:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77b80 x1853163985163392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319415.003528:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319415.003531:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319415.003533:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319415.003537:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4658 00000100:00080000:3.0:1767319415.003540:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319415.003544:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319415.003559:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75880 x1853163985163520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319415.003614:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319415.003619:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319415.003621:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319415.003625:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4658 00000100:00080000:3.0:1767319415.003628:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319415.003631:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319415.003643:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76a00 x1853163985163776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319415.003674:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319415.003678:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319415.003679:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319415.003683:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4658 00000100:00080000:3.0:1767319415.003686:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319415.003690:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319415.003700:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985164032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319415.003751:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985164160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319415.003840:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319415.003933:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319415.003940:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985163904:0@lo:400:kworker.0 00000100:00100000:3.0:1767319415.003971:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319415.004211:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319389/real 1767319389] req@ffff8ce1af73c000 x1853163985156864/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319414 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319415.004238:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1af73c000 x1853163985156864/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319414 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319415.004252:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985163392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319415.004285:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319415.004310:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985163520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319415.004336:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319415.004361:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985163776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319415.004384:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319415.004401:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985164032:0@lo:38:kworker.0 00000100:00100000:2.0:1767319415.004422:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319415.004438:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319415.004450:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319415.004454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985156864:0@lo:250:kworker.0 00000100:00100000:2.0:1767319415.004494:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985164160 00000100:00100000:2.0:1767319415.004503:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985164160 00000100:00100000:2.0:1767319415.004570:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3029 00000100:00100000:2.0:1767319415.004582:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985164160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319415.004637:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985164160:12345-0@lo:400:kworker.0 Request processed in 58us (797us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319415.004645:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3029 00000100:00100000:2.0:1767319415.004672:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163904 00000100:00100000:2.0:1767319415.004675:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163904 00000100:00100000:2.0:1767319415.004697:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163392 00000100:00100000:2.0:1767319415.004700:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163392 00000100:00100000:2.0:1767319415.004712:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163776 00000100:00100000:2.0:1767319415.004714:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163776 00000100:00100000:2.0:1767319415.004722:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985164032 00000100:00100000:2.0:1767319415.004724:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985164032 00000100:00100000:2.0:1767319415.004732:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4605 00000100:00100000:2.0:1767319415.004738:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985163904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319415.004765:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985163904:12345-0@lo:400:kworker.0 Request processed in 28us (794us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319415.004771:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4605 00000100:00100000:2.0:1767319415.004780:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5076 00000100:00100000:2.0:1767319415.004784:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985163392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319415.004804:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319415.004833:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985163392:12345-0@lo:38:kworker.0 Request processed in 47us (551us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319415.004838:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5076 00000100:00100000:2.0:1767319415.004848:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5077 00000100:00100000:2.0:1767319415.004852:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985163776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319415.004860:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319415.004882:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985163776:12345-0@lo:38:kworker.0 Request processed in 29us (498us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319415.005014:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985164288:0@lo:400:kworker.0 00000100:00100000:1.0:1767319415.005048:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319415.005076:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319415.005081:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985163648:0@lo:400:kworker.0 00000100:00100000:1.0:1767319415.005100:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319415.005198:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985164288 00000100:00100000:1.0:1767319415.005202:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985164288 00000100:00100000:1.0:1767319415.005230:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3030 00000100:00100000:1.0:1767319415.005236:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985164288:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319415.005268:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985164288:12345-0@lo:400:kworker.0 Request processed in 32us (219us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319415.005274:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3030 00000100:00080000:1.0:1767319415.005291:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319415.005296:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319415.005298:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319415.005310:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163520 00000100:00100000:1.0:1767319415.005314:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163520 00000100:00100000:1.0:1767319415.005333:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1759 00000100:00100000:1.0:1767319415.005338:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985163520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319415.005347:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319415.005371:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985163520:12345-0@lo:38:kworker.0 Request processed in 32us (1034us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319415.005380:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1759 00000100:00100000:1.0:1767319415.005400:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985163648 00000100:00100000:1.0:1767319415.005404:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985163648 00000100:00100000:1.0:1767319415.005426:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4606 00000100:00100000:1.0:1767319415.005431:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985163648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319415.005459:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985163648:12345-0@lo:400:kworker.0 Request processed in 28us (358us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319415.005466:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4606 00000100:00100000:1.0:1767319415.005477:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5078 00000100:00100000:1.0:1767319415.005480:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985164032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319415.005489:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319415.005511:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985164032:12345-0@lo:38:kworker.0 Request processed in 29us (1088us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319415.005517:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5078 00000100:00080000:1.0:1767319415.005531:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319415.005558:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985163648:0@lo:400:kworker.0 00000100:00100000:1.0:1767319415.005568:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985164288:0@lo:400:kworker.0 00000100:00100000:1.0:1767319415.005598:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985163904:0@lo:400:kworker.0 00000100:00100000:1.0:1767319415.005608:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985164160:0@lo:400:kworker.0 00000100:00080000:1.0:1767319415.005659:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319415.005664:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319415.005667:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985163392:0@lo:38:kworker.0 00000100:00080000:1.0:1767319415.005675:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319415.005699:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319415.005701:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985163520:0@lo:38:kworker.0 00000100:00080000:1.0:1767319415.005712:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319415.005715:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319415.005718:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985163776:0@lo:38:kworker.0 00000100:00080000:1.0:1767319415.005727:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319415.005730:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319415.005733:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985164032:0@lo:38:kworker.0 00000100:00100000:2.0:1767319415.005840:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5077 00000100:00080000:3.0:1767319415.005936:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319415.005941:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319415.005944:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319415.005949:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4637 00000100:00080000:3.0:1767319415.005953:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319415.005959:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319415.005986:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75500 x1853163985164416/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319415.006145:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985164416:0@lo:250:kworker.0 00000400:00000100:1.0:1767319415.006192:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985164416 offset 0 length 520: 4 00010000:00010000:0.0:1767319415.643404:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319416.667480:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319417.563512:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985164544:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319417.563625:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319417.563808:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985164544 00000100:00100000:3.0:1767319417.563818:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985164544 00000100:00100000:3.0:1767319417.563920:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2196 00000100:00100000:3.0:1767319417.563934:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985164544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319417.563978:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319417.564038:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985164544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (412us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319417.564047:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2196 00000100:00100000:2.0:1767319417.564115:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985164544:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319417.564203:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985164672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319417.564246:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319417.564272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319417.564339:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985164672 00000100:00100000:2.0:1767319417.564344:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985164672 00000100:00100000:2.0:1767319417.564392:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2197 00000100:00100000:2.0:1767319417.564401:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985164672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319417.564423:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319417.564472:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985164672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (225us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319417.564480:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2197 00000100:00100000:3.0:1767319417.564605:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985164672:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319417.564662:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319417.691655:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319418.715678:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319419.740315:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319420.074866:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319420.074983:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920559744 00000100:00100000:0.0:1767319420.074994:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920559744 00000100:00100000:0.0:1767319420.075122:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4607 00000100:00100000:0.0:1767319420.075134:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920559744:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319420.075257:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920559744:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 124us (400us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319420.075265:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4607 00000100:00100000:0.0:1767319420.075785:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319420.076066:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920559616 00000100:00100000:3.0:1767319420.076073:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920559616 00000100:00100000:3.0:1767319420.076107:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5079 00000100:00100000:3.0:1767319420.076114:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920559616:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319420.076138:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319420.076204:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920559616:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 88us (420us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319420.076210:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5079 00000100:00080000:1.0:1767319420.076641:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319420.123536:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319420.123546:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319420.123549:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319420.123553:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319420.123558:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319420.123566:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4663 00000100:00080000:0.0:1767319420.123570:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319420.123577:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319420.123615:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2300 x1853163985164800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319420.123667:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985164800:0@lo:38:kworker.0 00000100:00100000:1.0:1767319420.123733:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319420.123848:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985164800 00000100:00100000:2.0:1767319420.123857:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985164800 00000100:00100000:2.0:1767319420.123894:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5080 00000100:00100000:2.0:1767319420.123901:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985164800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319420.123919:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319420.124206:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319420.124210:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319420.124213:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319420.124217:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4663 00000100:00080000:0.0:1767319420.124220:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319420.124224:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319420.124242:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2680 x1853163985164928/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319420.124482:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985164928:0@lo:38:kworker.0 00000100:00100000:1.0:1767319420.124518:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319420.124585:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985164928 00000100:00100000:3.0:1767319420.124591:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985164928 00000100:00100000:3.0:1767319420.124617:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1760 00000100:00100000:3.0:1767319420.124624:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985164928:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319420.124636:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319420.124666:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985164928:12345-0@lo:38:kworker.0 Request processed in 41us (146us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319420.124672:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1760 00000100:00100000:2.0:1767319420.127114:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985164800:12345-0@lo:38:kworker.0 Request processed in 3203us (3371us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319420.127129:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5080 00000100:00080000:0.0:1767319420.128070:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319420.128076:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319420.128080:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319420.128086:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4663 00000100:00080000:0.0:1767319420.128090:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319420.128096:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319420.128116:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c380 x1853163985165184/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319420.128137:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319420.128140:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319420.128142:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319420.128145:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4663 00000100:00080000:0.0:1767319420.128147:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319420.128150:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319420.128160:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985165440/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319420.128225:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985165696:0@lo:400:kworker.0 00000100:00100000:0.0:1767319420.128283:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319420.128317:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319420.128324:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985165056:0@lo:400:kworker.0 00000100:00100000:0.0:1767319420.128347:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319420.128372:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165696 00000100:00100000:0.0:1767319420.128379:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165696 00000100:00100000:0.0:1767319420.128435:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3031 00000100:00100000:0.0:1767319420.128472:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985165696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319420.128520:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985165696:12345-0@lo:400:kworker.0 Request processed in 50us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319420.128528:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3031 00000100:00100000:0.0:1767319420.128570:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985165696:0@lo:400:kworker.0 00000100:00080000:0.0:1767319420.128605:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319420.129086:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985165568:0@lo:400:kworker.0 00000100:00100000:2.0:1767319420.129119:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319420.129147:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319420.129153:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985165312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319420.129318:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165056 00000100:00100000:3.0:1767319420.129323:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165056 00000100:00100000:3.0:1767319420.129360:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4608 00000100:00100000:3.0:1767319420.129368:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985165056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319420.129407:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985165056:12345-0@lo:400:kworker.0 Request processed in 40us (1059us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319420.129412:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4608 00000100:00100000:2.0:1767319420.129560:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319420.129630:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165312 00000100:00100000:3.0:1767319420.129634:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165312 00000100:00100000:3.0:1767319420.129668:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4609 00000100:00100000:3.0:1767319420.129675:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985165312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319420.129709:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985165312:12345-0@lo:400:kworker.0 Request processed in 34us (148us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319420.129715:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4609 00000100:00100000:2.0:1767319420.129794:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985165312:0@lo:400:kworker.0 00000100:00100000:0.0:1767319420.130041:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165568 00000100:00100000:0.0:1767319420.130046:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165568 00000100:00100000:0.0:1767319420.130256:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3032 00000100:00100000:0.0:1767319420.130263:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985165568:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319420.130367:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319420.130371:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319420.130396:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985165056:0@lo:400:kworker.0 00000100:00100000:0.0:1767319420.130946:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985165568:12345-0@lo:400:kworker.0 Request processed in 681us (1825us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319420.130955:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3032 00000100:00080000:1.0:1767319420.132230:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319420.132242:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319420.132247:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985164800:0@lo:38:kworker.0 00000100:00100000:1.0:1767319420.132298:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985165184:0@lo:38:kworker.0 00000100:00100000:1.0:1767319420.132362:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319420.132458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985165440:0@lo:38:kworker.0 00000100:00100000:1.0:1767319420.132483:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319420.132560:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985165568:0@lo:400:kworker.0 00000100:00080000:1.0:1767319420.137897:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319420.137910:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319420.137915:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985164928:0@lo:38:kworker.0 00000100:00100000:2.0:1767319420.142390:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165184 00000100:00100000:2.0:1767319420.142406:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165184 00000100:00100000:2.0:1767319420.142472:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165440 00000100:00100000:2.0:1767319420.142475:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165440 00000100:00100000:2.0:1767319420.142513:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5081 00000100:00100000:2.0:1767319420.142521:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985165184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319420.142545:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319420.142638:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985165184:12345-0@lo:38:kworker.0 Request processed in 115us (10273us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319420.142646:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5081 00000100:00100000:2.0:1767319420.142663:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5082 00000100:00100000:2.0:1767319420.142668:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985165440:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319420.142677:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319420.142701:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985165440:12345-0@lo:38:kworker.0 Request processed in 32us (10217us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319420.142707:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5082 00000100:00080000:1.0:1767319420.143493:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319420.143499:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319420.143503:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985165184:0@lo:38:kworker.0 00000100:00080000:1.0:1767319420.143513:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319420.143516:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319420.143519:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985165440:0@lo:38:kworker.0 00010000:00010000:2.0:1767319420.764075:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319421.787453:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319422.683406:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985165824:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319422.683464:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319422.683587:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165824 00000100:00100000:0.0:1767319422.683595:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165824 00000100:00080000:1.0:1767319422.683689:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319422.685398:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2198 00000100:00100000:0.0:1767319422.685420:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985165824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319422.685457:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319422.685553:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985165824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 142us (2087us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319422.685561:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2198 00000100:00100000:1.0:1767319422.685672:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985165824:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319422.687314:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985165952:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319422.687385:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319422.691800:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985165952 00000100:00100000:3.0:1767319422.691814:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985165952 00000100:00100000:3.0:1767319422.691884:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2199 00000100:00100000:3.0:1767319422.691894:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985165952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319422.691926:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319422.691999:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985165952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (4614us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319422.692007:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2199 00000100:00080000:2.0:1767319422.693196:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319422.693257:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985165952:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319422.819305:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319423.840388:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319424.859577:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319425.206263:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319425.206321:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319425.206423:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920560000 00000100:00100000:1.0:1767319425.206437:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920560000 00000100:00100000:1.0:1767319425.206521:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920559872 00000100:00100000:1.0:1767319425.206525:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920559872 00000100:00100000:1.0:1767319425.206550:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4610 00000100:00100000:1.0:1767319425.206563:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920560000:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319425.206658:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920560000:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (412us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319425.206669:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4610 00000100:00100000:1.0:1767319425.206692:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5083 00000100:00100000:1.0:1767319425.206696:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920559872:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319425.206724:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319425.206750:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920559872:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (430us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319425.206757:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5083 00000100:00080000:2.0:1767319425.207186:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319425.243320:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319425.243335:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319425.243339:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319425.243343:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319425.243348:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319425.243358:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4668 00000100:00080000:0.0:1767319425.243363:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319425.243372:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319425.243436:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1c00 x1853163985166080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319425.243466:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319425.243469:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319425.243471:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319425.243474:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4668 00000100:00080000:0.0:1767319425.243477:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319425.243481:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319425.243492:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985166208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319425.243539:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319425.243543:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319425.243545:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319425.243548:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4668 00000100:00080000:0.0:1767319425.243551:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319425.243555:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319425.243565:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c380 x1853163985166464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319425.243592:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985166336:0@lo:400:kworker.0 00000100:00080000:0.0:1767319425.243602:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319425.243606:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319425.243608:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319425.243612:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4668 00000100:00080000:0.0:1767319425.243614:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319425.243617:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319425.243628:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985166720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319425.243672:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985166592:0@lo:400:kworker.0 00000100:00100000:3.0:1767319425.243689:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319425.243712:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985166848:0@lo:400:kworker.0 00000100:00100000:0.0:1767319425.243749:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319425.243775:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319425.243785:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985166976:0@lo:400:kworker.0 00000100:00100000:0.0:1767319425.243797:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166848 00000100:00100000:0.0:1767319425.243806:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166848 00000100:00100000:3.0:1767319425.243814:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319425.243852:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166336 00000100:00100000:3.0:1767319425.243857:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166336 00000100:00100000:0.0:1767319425.243868:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166976 00000100:00100000:0.0:1767319425.243873:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166976 00000100:00100000:0.0:1767319425.243905:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3033 00000100:00100000:3.0:1767319425.243908:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4611 00000100:00100000:0.0:1767319425.243916:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985166848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319425.243922:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985166336:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319425.243993:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985166848:12345-0@lo:400:kworker.0 Request processed in 78us (243us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319425.244000:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985166336:12345-0@lo:400:kworker.0 Request processed in 81us (311us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319425.244006:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3033 00000100:00100000:3.0:1767319425.244014:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4611 00000100:00100000:0.0:1767319425.244033:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3034 00000100:00100000:0.0:1767319425.244042:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985166976:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319425.244051:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319425.244081:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985166976:12345-0@lo:400:kworker.0 Request processed in 39us (266us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319425.244085:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166592 00000100:00100000:3.0:1767319425.244090:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166592 00000100:00100000:0.0:1767319425.244095:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3034 00000100:00100000:3.0:1767319425.244121:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4612 00000100:00080000:0.0:1767319425.244124:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319425.244129:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985166592:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319425.244130:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319425.244135:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319425.244288:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985166848:0@lo:400:kworker.0 00000100:00100000:1.0:1767319425.244505:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985166080:0@lo:38:kworker.0 00000100:00100000:3.0:1767319425.244633:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985166592:12345-0@lo:400:kworker.0 Request processed in 504us (583us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319425.244641:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4612 00000100:00100000:2.0:1767319425.244966:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985166976:0@lo:400:kworker.0 00000100:00100000:2.0:1767319425.244982:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985166336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319425.246742:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319425.246821:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985166208:0@lo:38:kworker.0 00000100:00100000:1.0:1767319425.246861:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319425.246904:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985166464:0@lo:38:kworker.0 00000100:00100000:1.0:1767319425.246940:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319425.246958:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985166720:0@lo:38:kworker.0 00000100:00100000:1.0:1767319425.246979:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319425.247046:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166080 00000100:00100000:1.0:1767319425.247055:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166080 00000100:00100000:1.0:1767319425.247090:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166464 00000100:00100000:1.0:1767319425.247093:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166464 00000100:00100000:1.0:1767319425.247107:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166720 00000100:00100000:1.0:1767319425.247109:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166720 00000100:00100000:1.0:1767319425.247128:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5084 00000100:00100000:1.0:1767319425.247135:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985166080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319425.247182:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319425.247231:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985166080:12345-0@lo:38:kworker.0 Request processed in 94us (498us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319425.247238:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5084 00000100:00100000:1.0:1767319425.247256:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5085 00000100:00100000:1.0:1767319425.247260:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985166464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319425.247269:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319425.247292:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985166464:12345-0@lo:38:kworker.0 Request processed in 30us (351us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319425.247298:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5085 00000100:00100000:1.0:1767319425.247308:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5086 00000100:00100000:1.0:1767319425.247312:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985166720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319425.247321:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319425.247342:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985166720:12345-0@lo:38:kworker.0 Request processed in 28us (362us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319425.247347:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5086 00000100:00100000:2.0:1767319425.247521:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985166208 00000100:00100000:2.0:1767319425.247527:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985166208 00000100:00100000:2.0:1767319425.247553:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1761 00000100:00100000:2.0:1767319425.247558:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985166208:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319425.247569:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319425.247598:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985166208:12345-0@lo:38:kworker.0 Request processed in 39us (737us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319425.247603:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1761 00000100:00080000:1.0:1767319425.247693:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319425.247700:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319425.247704:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985166080:0@lo:38:kworker.0 00000100:00080000:1.0:1767319425.247714:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319425.247718:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319425.247721:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985166208:0@lo:38:kworker.0 00000100:00080000:1.0:1767319425.247729:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319425.247732:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319425.247735:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985166464:0@lo:38:kworker.0 00000100:00080000:1.0:1767319425.247743:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319425.247746:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319425.247749:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985166720:0@lo:38:kworker.0 00000100:00100000:0.0:1767319425.247871:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985166592:0@lo:400:kworker.0 00010000:00010000:1.0:1767319425.883496:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319426.907517:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319427.803961:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985167104:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319427.803989:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319427.804006:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985167232:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319427.804061:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319427.804130:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319427.804237:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985167232 00000100:00100000:2.0:1767319427.804247:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985167232 00000100:00100000:3.0:1767319427.804280:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985167104 00000100:00100000:3.0:1767319427.804287:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985167104 00000100:00100000:3.0:1767319427.804339:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2200 00000100:00100000:3.0:1767319427.804351:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985167104:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319427.804360:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319427.804367:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:3.0:1767319427.804449:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319427.804521:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985167104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 172us (390us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319427.804534:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2200 00000100:00100000:3.0:1767319427.804557:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2201 00000100:00100000:3.0:1767319427.804563:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985167232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319427.804592:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319427.804625:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985167232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (565us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319427.804632:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2201 00000100:00100000:1.0:1767319427.807944:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985167104:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319427.808048:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985167232:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319427.936473:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319428.955494:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319429.980002:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319430.321347:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319430.321406:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319430.321573:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920560256 00000100:00100000:0.0:1767319430.321588:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920560256 00000100:00100000:0.0:1767319430.321680:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920560128 00000100:00100000:0.0:1767319430.321684:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920560128 00000100:00100000:0.0:1767319430.321713:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4613 00000100:00100000:0.0:1767319430.321725:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920560256:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319430.321828:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920560256:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 104us (494us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319430.321870:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4613 00000100:00100000:1.0:1767319430.321891:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5087 00000100:00100000:1.0:1767319430.321900:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920560128:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319430.321928:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319430.321965:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920560128:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 64us (559us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319430.321973:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5087 00000100:00080000:2.0:1767319430.322197:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319430.364501:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319430.364516:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319430.364519:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319430.364524:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319430.364530:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319430.364540:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4673 00000100:00080000:0.0:1767319430.364546:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319430.364555:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319430.364611:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb480 x1853163985167360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319430.364657:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319430.364662:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319430.364664:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319430.364668:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4673 00000100:00080000:0.0:1767319430.364670:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319430.364674:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319430.364689:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bea300 x1853163985167488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319430.364741:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319430.364746:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319430.364749:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319430.364754:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4673 00000100:00080000:0.0:1767319430.364757:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319430.364761:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319430.364775:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985167360:0@lo:38:kworker.0 00000100:00100000:0.0:1767319430.364777:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9c00 x1853163985167744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319430.364813:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319430.364817:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319430.364819:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319430.364823:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4673 00000100:00080000:0.0:1767319430.364825:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319430.364829:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319430.364851:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8000 x1853163985168000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319430.364892:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.364969:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985167488:0@lo:38:kworker.0 00000100:00100000:1.0:1767319430.365001:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.365046:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985167744:0@lo:38:kworker.0 00000100:00100000:1.0:1767319430.365074:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.365095:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985168000:0@lo:38:kworker.0 00000100:00100000:1.0:1767319430.365118:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.365155:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985168128:0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.365226:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.365249:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319430.365255:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985167872:0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.365276:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.365311:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985167360 00000100:00100000:1.0:1767319430.365321:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985167360 00000100:00100000:1.0:1767319430.365361:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985167744 00000100:00100000:1.0:1767319430.365364:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985167744 00000100:00100000:1.0:1767319430.365378:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168000 00000100:00100000:1.0:1767319430.365381:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168000 00000100:00100000:1.0:1767319430.365389:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985167872 00000100:00100000:1.0:1767319430.365392:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985167872 00000100:00100000:1.0:1767319430.365431:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4614 00000100:00100000:1.0:1767319430.365442:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985167872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.365501:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985167872:12345-0@lo:400:kworker.0 Request processed in 60us (222us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319430.365510:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4614 00000100:00100000:1.0:1767319430.365534:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5088 00000100:00100000:3.0:1767319430.365540:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985168256:0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.365541:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985167360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319430.365568:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319430.365584:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.365615:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985167360:12345-0@lo:38:kworker.0 Request processed in 73us (724us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319430.365619:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319430.365624:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5088 00000100:00100000:3.0:1767319430.365628:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985167616:0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.365640:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5089 00000100:00100000:1.0:1767319430.365646:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985167744:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319430.365655:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767319430.365658:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319430.365679:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168128 00000100:00100000:3.0:1767319430.365685:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168128 00000100:00100000:1.0:1767319430.365697:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985167744:12345-0@lo:38:kworker.0 Request processed in 49us (623us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319430.365705:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5089 00000100:00100000:1.0:1767319430.365718:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985167616 00000100:00100000:1.0:1767319430.365723:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985167616 00000100:00100000:3.0:1767319430.365729:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168256 00000100:00100000:3.0:1767319430.365734:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168256 00000100:00100000:1.0:1767319430.365747:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4615 00000100:00100000:1.0:1767319430.365756:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985167616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319430.365768:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3035 00000100:00100000:3.0:1767319430.365777:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985168128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.365800:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985167616:12345-0@lo:400:kworker.0 Request processed in 45us (144us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319430.365808:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4615 00000100:00100000:3.0:1767319430.365824:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985168128:12345-0@lo:400:kworker.0 Request processed in 48us (596us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319430.365827:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5090 00000100:00100000:3.0:1767319430.365846:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3035 00000100:00100000:1.0:1767319430.365848:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985168000:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319430.365860:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319430.365880:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3036 00000100:00100000:1.0:1767319430.365891:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985168000:12345-0@lo:38:kworker.0 Request processed in 41us (770us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319430.365892:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985168256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.365905:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5090 00000100:00100000:3.0:1767319430.365935:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985168256:12345-0@lo:400:kworker.0 Request processed in 46us (350us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319430.365945:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3036 00000100:00100000:1.0:1767319430.365972:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985167872:0@lo:400:kworker.0 00000100:00100000:3.0:1767319430.365989:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985167488 00000100:00100000:3.0:1767319430.365995:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985167488 00000100:00100000:1.0:1767319430.366001:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985168128:0@lo:400:kworker.0 00000100:00100000:3.0:1767319430.366019:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1762 00000100:00100000:3.0:1767319430.366025:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985167488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319430.366037:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319430.366044:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985167616:0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.366059:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985168256:0@lo:400:kworker.0 00000100:00100000:3.0:1767319430.366073:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985167488:12345-0@lo:38:kworker.0 Request processed in 45us (1070us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319430.366081:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1762 00000100:00080000:1.0:1767319430.366090:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.366098:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.366102:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985167744:0@lo:38:kworker.0 00000100:00080000:1.0:1767319430.366112:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.366115:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.366120:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985168000:0@lo:38:kworker.0 00000100:00080000:1.0:1767319430.366129:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.366133:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.366137:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985167360:0@lo:38:kworker.0 00000100:00080000:1.0:1767319430.366147:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.366151:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.366155:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985167488:0@lo:38:kworker.0 00000100:00080000:2.0:1767319430.366579:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319430.366584:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319430.366587:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319431.003756:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319432.027429:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319432.927212:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319432.927244:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985168384:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319432.927336:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319432.927586:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168384 00000100:00100000:1.0:1767319432.927597:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168384 00000100:00100000:1.0:1767319432.927675:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2202 00000100:00100000:1.0:1767319432.927685:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985168384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319432.927725:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319432.927781:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985168384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (445us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319432.927818:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2202 00000100:00080000:0.0:1767319432.929858:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319432.929950:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985168384:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319432.930306:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319432.930325:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985168512:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319432.930404:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319432.930691:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168512 00000100:00100000:2.0:1767319432.930701:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168512 00000100:00100000:2.0:1767319432.930781:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2203 00000100:00100000:2.0:1767319432.930804:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985168512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319432.930842:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319432.930911:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985168512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 111us (505us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319432.930919:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2203 00000100:00080000:0.0:1767319432.930992:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319432.931050:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985168512:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319433.052875:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319434.077865:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319435.099464:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319435.435699:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319435.435804:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920560512 00000100:00100000:1.0:1767319435.435817:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920560512 00000100:00100000:1.0:1767319435.435916:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4616 00000100:00100000:1.0:1767319435.435928:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920560512:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319435.436039:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920560512:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 112us (363us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319435.436047:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4616 00000100:00080000:0.0:1767319435.436119:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319435.437813:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319435.437911:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920560384 00000100:00100000:1.0:1767319435.437920:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920560384 00000100:00100000:1.0:1767319435.437965:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5091 00000100:00100000:1.0:1767319435.437975:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920560384:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319435.438005:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319435.438069:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920560384:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (266us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319435.438077:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5091 00000100:00080000:0.0:1767319435.483531:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319435.483544:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319435.483547:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319435.483551:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319435.483557:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319435.483567:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4678 00000100:00080000:0.0:1767319435.483573:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319435.483580:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319435.483633:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beaa00 x1853163985168640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319435.483668:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319435.483672:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319435.483674:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319435.483677:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4678 00000100:00080000:0.0:1767319435.483680:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319435.483684:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319435.483697:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9180 x1853163985168768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319435.483757:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319435.483761:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319435.483763:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319435.483767:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4678 00000100:00080000:0.0:1767319435.483769:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319435.483772:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319435.483785:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb480 x1853163985169024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319435.483801:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985168896:0@lo:400:kworker.0 00000100:00080000:0.0:1767319435.483818:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319435.483823:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319435.483826:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319435.483830:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4678 00000100:00080000:0.0:1767319435.483833:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319435.483837:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319435.483852:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9c00 x1853163985169280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319435.483928:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985169408:0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.483934:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985169152:0@lo:400:kworker.0 00000100:00100000:2.0:1767319435.483946:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319435.484023:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319435.484030:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985168640:0@lo:38:kworker.0 00000100:00100000:2.0:1767319435.484036:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985169536:0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.484157:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319435.484260:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168896 00000100:00100000:0.0:1767319435.484270:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168896 00000100:00100000:0.0:1767319435.484346:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169152 00000100:00100000:0.0:1767319435.484349:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169152 00000100:00100000:0.0:1767319435.484374:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4617 00000100:00100000:0.0:1767319435.484383:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985168896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.484440:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985168896:12345-0@lo:400:kworker.0 Request processed in 58us (504us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319435.484448:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4617 00000100:00100000:0.0:1767319435.484469:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4618 00000100:00100000:0.0:1767319435.484476:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985169152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.484509:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985169152:12345-0@lo:400:kworker.0 Request processed in 34us (353us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319435.484516:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4618 00000100:00080000:0.0:1767319435.484534:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319435.484568:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985169152:0@lo:400:kworker.0 00000100:00100000:3.0:1767319435.486811:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319435.488310:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169408 00000100:00100000:0.0:1767319435.488323:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169408 00000100:00100000:0.0:1767319435.488532:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3037 00000100:00100000:0.0:1767319435.488544:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985169408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.488628:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985169408:12345-0@lo:400:kworker.0 Request processed in 86us (1824us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319435.488713:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985169408:0@lo:400:kworker.0 00000100:00080000:3.0:1767319435.488770:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319435.488862:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3037 00000100:00100000:1.0:1767319435.490149:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319435.490256:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985168768:0@lo:38:kworker.0 00000100:00100000:1.0:1767319435.490395:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319435.490433:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985169024:0@lo:38:kworker.0 00000100:00100000:1.0:1767319435.490461:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319435.490488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985169280:0@lo:38:kworker.0 00000100:00100000:1.0:1767319435.490509:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319435.490565:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168640 00000100:00100000:1.0:1767319435.490574:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168640 00000100:00100000:3.0:1767319435.490591:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985168768 00000100:00100000:3.0:1767319435.490606:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985168768 00000100:00100000:1.0:1767319435.490614:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169024 00000100:00100000:1.0:1767319435.490620:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169024 00000100:00100000:1.0:1767319435.490663:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169280 00000100:00100000:1.0:1767319435.490667:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169280 00000100:00100000:3.0:1767319435.490678:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1763 00000100:00100000:3.0:1767319435.490689:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985168768:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319435.490697:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5092 00000100:00100000:1.0:1767319435.490705:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985168640:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319435.490716:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767319435.490726:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319435.490821:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985168768:12345-0@lo:38:kworker.0 Request processed in 130us (422us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319435.490830:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1763 00000100:00100000:0.0:1767319435.490843:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5093 00000100:00100000:0.0:1767319435.490852:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985169024:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319435.490864:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319435.490907:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985169024:12345-0@lo:38:kworker.0 Request processed in 55us (445us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319435.491401:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5094 00000100:00100000:3.0:1767319435.491410:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985169280:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319435.491425:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319435.491469:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985169280:12345-0@lo:38:kworker.0 Request processed in 59us (959us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319435.492451:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985168640:12345-0@lo:38:kworker.0 Request processed in 1738us (2304us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319435.492507:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5093 00000100:00080000:0.0:1767319435.492606:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319435.492612:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319435.492616:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985168640:0@lo:38:kworker.0 00000100:00080000:0.0:1767319435.492630:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319435.492633:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319435.492636:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985168768:0@lo:38:kworker.0 00000100:00080000:0.0:1767319435.492647:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319435.492651:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319435.492654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985169024:0@lo:38:kworker.0 00000100:00080000:0.0:1767319435.492664:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319435.492668:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319435.492671:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985169280:0@lo:38:kworker.0 00000100:00100000:3.0:1767319435.493099:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5094 00000100:00100000:2.0:1767319435.499028:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319435.499141:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985168896:0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.500714:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169536 00000100:00100000:0.0:1767319435.500730:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169536 00000100:00100000:0.0:1767319435.500857:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3038 00000100:00100000:0.0:1767319435.500870:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985169536:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.500982:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985169536:12345-0@lo:400:kworker.0 Request processed in 115us (1967us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319435.500990:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3038 00000100:00100000:2.0:1767319435.501208:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985169536:0@lo:400:kworker.0 00000100:00080000:3.0:1767319435.501208:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319435.501286:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5092 00010000:00010000:1.0:1767319436.123416:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319437.147474:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319438.043887:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985169664:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319438.044000:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319438.044363:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985169792:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319438.044401:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319438.044569:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169664 00000100:00100000:2.0:1767319438.044580:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169664 00000100:00100000:2.0:1767319438.044655:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169792 00000100:00100000:2.0:1767319438.044659:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169792 00000100:00100000:2.0:1767319438.044686:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2204 00000100:00100000:2.0:1767319438.044708:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985169664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319438.044750:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319438.044802:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985169664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (801us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319438.044809:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2204 00000100:00100000:2.0:1767319438.044829:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2205 00000100:00100000:2.0:1767319438.044836:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985169792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319438.044853:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319438.044882:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985169792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (481us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319438.044889:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2205 00000100:00100000:2.0:1767319438.044957:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985169664:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319438.045131:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319438.045136:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319438.045189:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985169792:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319438.171462:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319439.196106:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319439.530254:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319439.530315:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920560640 offset 0 length 520: 4 00000100:00100000:2.0:1767319439.530350:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319439.530586:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920560896 00000100:00100000:3.0:1767319439.530604:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920560896 00000100:00100000:3.0:1767319439.530707:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920560768 00000100:00100000:3.0:1767319439.530710:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920560768 00000100:00100000:3.0:1767319439.530732:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4619 00000100:00100000:3.0:1767319439.530744:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920560896:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319439.530852:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920560896:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (616us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319439.530861:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4619 00000100:00100000:3.0:1767319439.530882:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5095 00000100:00100000:3.0:1767319439.530887:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920560768:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319439.530914:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319439.530941:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920560768:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (590us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319439.530949:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5095 00000100:00080000:0.0:1767319439.531340:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319440.219483:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319440.603314:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319440.603330:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319440.603330:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319415/real 1767319415] req@ffff8ce19db75500 x1853163985164416/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319440 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319440.603333:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319440.603341:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319440.603347:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319440.603355:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4683 00000100:00080000:0.0:1767319440.603360:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319440.603370:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319440.603377:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19db75500 x1853163985164416/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319440 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319440.603400:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319440.603424:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9c00 x1853163985169920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319440.603429:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319440.603437:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985164416:0@lo:250:kworker.0 00000100:00080000:0.0:1767319440.603450:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319440.603455:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319440.603457:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319440.603463:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4683 00000100:00080000:0.0:1767319440.603466:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319440.603470:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319440.603474:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985169920:0@lo:38:kworker.0 00000100:00100000:0.0:1767319440.603487:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb480 x1853163985170048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319440.603540:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319440.603545:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319440.603547:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319440.603551:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4683 00000100:00080000:0.0:1767319440.603554:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319440.603559:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319440.603563:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319440.603577:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beaa00 x1853163985170304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319440.603614:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319440.603618:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319440.603620:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319440.603625:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4683 00000100:00080000:0.0:1767319440.603627:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319440.603631:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319440.603654:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985170048:0@lo:38:kworker.0 00000100:00100000:0.0:1767319440.603660:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb800 x1853163985170560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319440.603693:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319440.603705:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319440.603710:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319440.603712:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319440.603716:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4663 00000100:00080000:0.0:1767319440.603719:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319440.603722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985170304:0@lo:38:kworker.0 00000100:00080000:0.0:1767319440.603724:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319440.603743:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bebb80 x1853163985170944/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319440.603754:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319440.603774:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985170688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319440.603824:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319440.603892:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985170048 00000100:00100000:3.0:1767319440.603903:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985170048 00000100:00100000:3.0:1767319440.603959:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1764 00000100:00100000:3.0:1767319440.603969:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985170048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319440.603993:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319440.604042:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985170048:12345-0@lo:38:kworker.0 Request processed in 73us (348us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319440.604049:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1764 00000100:00100000:3.0:1767319440.604074:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985170816:0@lo:400:kworker.0 00000100:00100000:3.0:1767319440.604125:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985170176:0@lo:400:kworker.0 00000100:00100000:1.0:1767319440.604782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985170560:0@lo:38:kworker.0 00000100:00100000:1.0:1767319440.604814:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319440.604834:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985170944:0@lo:250:kworker.0 00000400:00000100:1.0:1767319440.604870:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985170944 offset 0 length 520: 4 00000100:00080000:1.0:1767319440.604911:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319440.604916:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319440.604920:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985170048:0@lo:38:kworker.0 00000100:00100000:1.0:1767319440.604973:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985169920 00000100:00100000:1.0:1767319440.605092:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985169920 00000100:00100000:1.0:1767319440.605125:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985170304 00000100:00100000:1.0:1767319440.605128:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985170304 00000100:00100000:1.0:1767319440.605145:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985170560 00000100:00100000:1.0:1767319440.605148:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985170560 00000100:00100000:1.0:1767319440.605182:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5096 00000100:00100000:1.0:1767319440.605190:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985169920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319440.605208:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319440.605249:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985169920:12345-0@lo:38:kworker.0 Request processed in 59us (1689us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319440.605256:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5096 00000100:00100000:1.0:1767319440.605272:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5097 00000100:00100000:1.0:1767319440.605277:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985170304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319440.605286:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319440.605308:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985170304:12345-0@lo:38:kworker.0 Request processed in 29us (1555us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319440.605313:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5097 00000100:00100000:1.0:1767319440.605325:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5098 00000100:00100000:1.0:1767319440.605328:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985170560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319440.605336:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319440.605358:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985170560:12345-0@lo:38:kworker.0 Request processed in 29us (543us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319440.605364:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5098 00000100:00080000:1.0:1767319440.605403:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319440.605407:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319440.605411:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985170560:0@lo:38:kworker.0 00000100:00080000:1.0:1767319440.605420:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319440.605424:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319440.605427:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985170304:0@lo:38:kworker.0 00000100:00080000:1.0:1767319440.605436:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319440.605440:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319440.605443:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985169920:0@lo:38:kworker.0 00000100:00100000:1.0:1767319440.605489:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985170432:0@lo:400:kworker.0 00000100:00100000:1.0:1767319440.605515:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319440.605547:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985170432 00000100:00100000:1.0:1767319440.605551:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985170432 00000100:00100000:1.0:1767319440.605594:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4620 00000100:00100000:1.0:1767319440.605602:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985170432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319440.607456:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985170432:12345-0@lo:400:kworker.0 Request processed in 1843us (1926us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319440.607472:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4620 00000100:00100000:1.0:1767319440.607562:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985170432:0@lo:400:kworker.0 00000100:00080000:1.0:1767319440.607594:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319440.608359:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985170688 00000100:00100000:0.0:1767319440.608372:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985170688 00000100:00100000:2.0:1767319440.608420:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319440.608467:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319440.608522:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319440.608570:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985170176 00000100:00100000:2.0:1767319440.608575:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985170176 00000100:00080000:3.0:1767319440.608607:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319440.608619:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4621 00000100:00100000:2.0:1767319440.608630:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985170176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319440.608726:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985170176:12345-0@lo:400:kworker.0 Request processed in 99us (204us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319440.608734:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4621 00000100:00100000:0.0:1767319440.609519:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985170816 00000100:00100000:0.0:1767319440.609528:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985170816 00000100:00100000:0.0:1767319440.609585:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3039 00000100:00100000:0.0:1767319440.609595:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985170688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319440.609677:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985170688:12345-0@lo:400:kworker.0 Request processed in 83us (5852us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319440.609685:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3039 00000100:00100000:0.0:1767319440.609701:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3040 00000100:00100000:0.0:1767319440.609707:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985170816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319440.609741:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985170816:12345-0@lo:400:kworker.0 Request processed in 35us (1325us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319440.609747:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3040 00000100:00080000:0.0:1767319440.609771:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319440.609809:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985170688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319440.609954:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985170816:0@lo:400:kworker.0 00000100:00100000:3.0:1767319440.610745:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985170176:0@lo:400:kworker.0 00010000:00010000:1.0:1767319441.245105:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319442.268152:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319443.163366:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985171072:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319443.163474:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319443.163648:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171072 00000100:00100000:1.0:1767319443.163660:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171072 00000100:00100000:1.0:1767319443.163759:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2206 00000100:00100000:1.0:1767319443.163771:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985171072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319443.163812:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767319443.163881:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319443.164047:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985171200:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319443.164272:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319443.164467:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171200 00000100:00100000:3.0:1767319443.164478:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171200 00000100:00100000:3.0:1767319443.164589:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2207 00000100:00100000:3.0:1767319443.164612:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb5050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985171200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319443.164654:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319443.164751:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb5050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985171200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 154us (489us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319443.164759:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2207 00000100:00100000:3.0:1767319443.164819:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985171200:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319443.166705:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985171072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2921us (3214us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319443.166712:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985171072:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319443.166922:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2206 00000100:00080000:2.0:1767319443.169696:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319443.292840:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319444.315898:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319444.651964:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319444.652139:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319444.652471:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561152 00000100:00100000:2.0:1767319444.652481:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561152 00000100:00100000:2.0:1767319444.652564:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561024 00000100:00100000:2.0:1767319444.652577:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561024 00000100:00100000:2.0:1767319444.652599:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4622 00000100:00100000:2.0:1767319444.652610:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920561152:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319444.652702:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920561152:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 94us (747us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319444.652711:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4622 00000100:00100000:2.0:1767319444.652730:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5099 00000100:00100000:2.0:1767319444.652735:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920561024:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319444.652754:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319444.652779:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920561024:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (641us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319444.652786:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5099 00000100:00080000:0.0:1767319444.653243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319445.340134:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319445.723499:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319445.723511:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319445.723514:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319445.723519:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319445.723523:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319445.723529:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4688 00000100:00080000:3.0:1767319445.723533:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319445.723538:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319445.723591:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc700 x1853163985171328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319445.723608:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319445.723611:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319445.723613:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319445.723616:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4688 00000100:00080000:3.0:1767319445.723619:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767319445.723621:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985171328:0@lo:38:kworker.0 00000100:00080000:3.0:1767319445.723632:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319445.723643:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffd180 x1853163985171456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319445.723689:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319445.723699:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319445.723702:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319445.723704:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319445.723707:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4688 00000100:00080000:3.0:1767319445.723709:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319445.723712:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319445.723722:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff100 x1853163985171712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319445.723743:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319445.723747:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985171456:0@lo:38:kworker.0 00000100:00080000:3.0:1767319445.723747:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319445.723749:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319445.723752:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4688 00000100:00080000:3.0:1767319445.723754:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319445.723757:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319445.723767:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985171968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319445.723772:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319445.723795:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985171712:0@lo:38:kworker.0 00000100:00100000:2.0:1767319445.723813:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319445.723814:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985172224:0@lo:400:kworker.0 00000100:00100000:2.0:1767319445.723828:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985171968:0@lo:38:kworker.0 00000100:00100000:3.0:1767319445.723841:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319445.723845:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319445.723859:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319445.723864:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985171584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319445.723870:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171328 00000100:00100000:2.0:1767319445.723876:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171328 00000100:00100000:3.0:1767319445.723883:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319445.723902:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171712 00000100:00100000:2.0:1767319445.723905:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171712 00000100:00100000:2.0:1767319445.723916:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171968 00000100:00100000:3.0:1767319445.723916:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171456 00000100:00100000:2.0:1767319445.723918:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171968 00000100:00100000:3.0:1767319445.723918:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171456 00000100:00100000:2.0:1767319445.723940:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171584 00000100:00100000:3.0:1767319445.723940:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1765 00000100:00100000:2.0:1767319445.723942:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171584 00000100:00100000:3.0:1767319445.723946:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985171456:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319445.723966:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319445.723969:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4623 00000100:00100000:2.0:1767319445.723976:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985171584:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319445.724004:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985171456:12345-0@lo:38:kworker.0 Request processed in 56us (229us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319445.724010:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1765 00000100:00100000:2.0:1767319445.724011:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985171584:12345-0@lo:400:kworker.0 Request processed in 37us (128us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319445.724019:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4623 00000100:00100000:2.0:1767319445.724031:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5100 00000100:00100000:3.0:1767319445.724032:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985171840:0@lo:400:kworker.0 00000100:00100000:2.0:1767319445.724035:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985171328:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319445.724044:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319445.724053:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319445.724061:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985171328:12345-0@lo:38:kworker.0 Request processed in 26us (373us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319445.724065:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319445.724067:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5100 00000100:00100000:3.0:1767319445.724069:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985172096:0@lo:400:kworker.0 00000100:00100000:2.0:1767319445.724075:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985171840 00000100:00100000:2.0:1767319445.724077:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985171840 00000100:00100000:3.0:1767319445.724088:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319445.724092:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4624 00000100:00100000:2.0:1767319445.724097:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985171840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319445.724125:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985171840:12345-0@lo:400:kworker.0 Request processed in 27us (71us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319445.724125:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5101 00000100:00100000:2.0:1767319445.724130:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4624 00000100:00100000:2.0:1767319445.724141:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5102 00000100:00100000:3.0:1767319445.724141:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985171712:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319445.724145:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985171968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319445.724151:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767319445.724153:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319445.724194:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985171968:12345-0@lo:38:kworker.0 Request processed in 47us (346us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319445.724194:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985171712:12345-0@lo:38:kworker.0 Request processed in 62us (379us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319445.724200:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5102 00000100:00100000:3.0:1767319445.724202:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5101 00000100:00100000:2.0:1767319445.724216:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172224 00000100:00100000:2.0:1767319445.724219:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172224 00000100:00080000:3.0:1767319445.724231:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319445.724235:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319445.724239:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985171968:0@lo:38:kworker.0 00000100:00100000:2.0:1767319445.724240:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172096 00000100:00100000:2.0:1767319445.724243:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172096 00000100:00080000:3.0:1767319445.724249:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319445.724252:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319445.724255:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985171456:0@lo:38:kworker.0 00000100:00100000:2.0:1767319445.724258:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3041 00000100:00080000:3.0:1767319445.724262:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767319445.724263:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985172224:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319445.724279:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319445.724282:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985171712:0@lo:38:kworker.0 00000100:00080000:3.0:1767319445.724291:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767319445.724293:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985172224:12345-0@lo:400:kworker.0 Request processed in 30us (452us total) trans 0 rc 0/0 00000100:00080000:3.0:1767319445.724294:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319445.724296:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985171328:0@lo:38:kworker.0 00000100:00100000:2.0:1767319445.724298:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3041 00000100:00100000:2.0:1767319445.724308:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3042 00000100:00100000:2.0:1767319445.724313:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985172096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319445.724337:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985171584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319445.724346:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985172224:0@lo:400:kworker.0 00000100:00100000:2.0:1767319445.724350:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985172096:12345-0@lo:400:kworker.0 Request processed in 38us (262us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319445.724355:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3042 00000100:00100000:3.0:1767319445.724371:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985172096:0@lo:400:kworker.0 00000100:00100000:3.0:1767319445.724379:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985171840:0@lo:400:kworker.0 00000100:00080000:0.0:1767319445.725501:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319445.725507:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319445.725510:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319446.363492:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319447.387966:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319448.283771:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985172352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319448.283862:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319448.284342:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172352 00000100:00100000:2.0:1767319448.284351:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172352 00000100:00100000:2.0:1767319448.284433:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2208 00000100:00100000:2.0:1767319448.284442:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985172352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319448.284479:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319448.284537:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985172352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (673us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319448.284544:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2208 00000100:00100000:1.0:1767319448.284638:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985172352:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319448.284702:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319448.284841:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985172480:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319448.284883:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319448.285277:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172480 00000100:00100000:3.0:1767319448.285283:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172480 00000100:00100000:3.0:1767319448.285329:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2209 00000100:00100000:3.0:1767319448.285336:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985172480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319448.285353:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319448.285391:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985172480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (507us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319448.285398:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2209 00000100:00080000:0.0:1767319448.285748:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319448.285776:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985172480:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319448.413343:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319449.435887:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319449.772155:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319449.772237:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319449.772322:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561408 00000100:00100000:1.0:1767319449.772331:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561408 00000100:00100000:1.0:1767319449.772401:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561280 00000100:00100000:1.0:1767319449.772405:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561280 00000100:00100000:1.0:1767319449.772426:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4625 00000100:00100000:1.0:1767319449.772436:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920561408:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319449.772534:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920561408:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319449.772541:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4625 00000100:00100000:1.0:1767319449.772606:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5103 00000100:00100000:1.0:1767319449.772611:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920561280:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319449.772631:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319449.772656:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920561280:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (418us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319449.772662:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5103 00000100:00080000:0.0:1767319449.772754:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319450.459844:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319450.843251:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319450.843263:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319450.843266:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319450.843270:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319450.843275:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319450.843283:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4693 00000100:00080000:3.0:1767319450.843289:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319450.843294:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319450.843336:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffed80 x1853163985172608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319450.843367:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319450.843370:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319450.843372:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319450.843377:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4693 00000100:00080000:3.0:1767319450.843378:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319450.843381:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319450.843394:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc000 x1853163985172736/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319450.843468:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985172864:0@lo:400:kworker.0 00000100:00080000:3.0:1767319450.843488:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319450.843494:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319450.843496:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319450.843499:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985172608:0@lo:38:kworker.0 00000100:00080000:3.0:1767319450.843502:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4693 00000100:00080000:3.0:1767319450.843509:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319450.843513:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319450.843528:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff100 x1853163985172992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319450.843566:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319450.843578:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985173120:0@lo:400:kworker.0 00000100:00100000:1.0:1767319450.843607:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319450.843640:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985172736:0@lo:38:kworker.0 00000100:00080000:3.0:1767319450.843649:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319450.843652:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319450.843654:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767319450.843658:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172608 00000100:00080000:3.0:1767319450.843658:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4693 00000100:00080000:3.0:1767319450.843662:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319450.843667:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319450.843668:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172608 00000100:00100000:2.0:1767319450.843674:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319450.843678:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985173248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319450.843712:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985173120 00000100:00100000:0.0:1767319450.843717:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985173120 00000100:00100000:2.0:1767319450.843718:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985172992:0@lo:38:kworker.0 00000100:00100000:3.0:1767319450.843724:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172736 00000100:00100000:3.0:1767319450.843730:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172736 00000100:00100000:2.0:1767319450.843750:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319450.843763:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1766 00000100:00100000:3.0:1767319450.843772:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985172736:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319450.843778:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172992 00000100:00100000:2.0:1767319450.843781:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985173248:0@lo:38:kworker.0 00000100:00100000:0.0:1767319450.843783:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172992 00010000:00020000:3.0:1767319450.843800:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319450.843812:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4626 00000100:00100000:0.0:1767319450.843824:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985173120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319450.843846:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985172736:12345-0@lo:38:kworker.0 Request processed in 73us (172us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319450.843855:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1766 00000100:00100000:0.0:1767319450.843874:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985173120:12345-0@lo:400:kworker.0 Request processed in 53us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319450.843882:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4626 00000100:00100000:3.0:1767319450.843901:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5104 00000100:00100000:3.0:1767319450.843907:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985172608:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319450.843915:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5105 00000100:00100000:2.0:1767319450.843918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767319450.843919:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319450.843926:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985172992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319450.843939:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319450.843953:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319450.843957:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985172608:12345-0@lo:38:kworker.0 Request processed in 47us (391us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319450.843962:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319450.843970:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985172736:0@lo:38:kworker.0 00000100:00100000:3.0:1767319450.843971:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5104 00000100:00100000:0.0:1767319450.843976:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985172992:12345-0@lo:38:kworker.0 Request processed in 49us (224us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319450.843985:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5105 00000100:00100000:3.0:1767319450.843990:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985173248 00000100:00100000:3.0:1767319450.843994:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985173248 00000100:00080000:2.0:1767319450.844006:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319450.844013:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319450.844019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985172992:0@lo:38:kworker.0 00000100:00100000:0.0:1767319450.844022:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5106 00000100:00100000:0.0:1767319450.844030:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985173248:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767319450.844033:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319450.844038:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:0.0:1767319450.844043:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319450.844054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985172608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319450.844078:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985173248:12345-0@lo:38:kworker.0 Request processed in 45us (265us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319450.844081:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319450.844087:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319450.844090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985173248:0@lo:38:kworker.0 00000100:00100000:2.0:1767319450.844129:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985173504:0@lo:400:kworker.0 00000100:00100000:1.0:1767319450.844134:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985173120:0@lo:400:kworker.0 00000100:00100000:1.0:1767319450.844149:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319450.844154:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985173376:0@lo:400:kworker.0 00000100:00100000:2.0:1767319450.844183:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319450.844210:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319450.844248:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319450.844265:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985173504 00000100:00100000:2.0:1767319450.844266:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319450.844271:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985173504 00000100:00100000:2.0:1767319450.844302:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985172864 00000100:00100000:2.0:1767319450.844307:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985172864 00000100:00100000:0.0:1767319450.844358:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5106 00000100:00080000:3.0:1767319450.844369:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319450.844374:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319450.844391:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3043 00000100:00100000:0.0:1767319450.844402:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985173504:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319450.844422:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985173376 00000100:00100000:1.0:1767319450.844426:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985173376 00000100:00100000:1.0:1767319450.844450:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3044 00000100:00100000:3.0:1767319450.844495:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985173504:0@lo:400:kworker.0 00000100:00080000:3.0:1767319450.844511:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319450.844539:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985173376:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319450.844597:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985173376:12345-0@lo:400:kworker.0 Request processed in 60us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319450.844602:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3044 00000100:00100000:2.0:1767319450.844619:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4627 00000100:00100000:2.0:1767319450.844629:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985172864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319450.844648:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985173376:0@lo:400:kworker.0 00000100:00100000:2.0:1767319450.844684:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985172864:12345-0@lo:400:kworker.0 Request processed in 56us (417us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319450.844693:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4627 00000100:00100000:1.0:1767319450.844699:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985172864:0@lo:400:kworker.0 00000100:00100000:0.0:1767319450.844728:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985173504:12345-0@lo:400:kworker.0 Request processed in 54us (292us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319450.844738:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3043 00010000:00010000:2.0:1767319451.485767:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319452.507344:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319453.404486:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985173632:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319453.404567:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319453.404626:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985173632 00000100:00100000:1.0:1767319453.404633:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985173632 00000100:00100000:1.0:1767319453.404683:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2210 00000100:00100000:1.0:1767319453.404691:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985173632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319453.404720:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319453.404902:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985173760:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319453.404928:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319453.404986:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985173760 00000100:00100000:0.0:1767319453.404990:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985173760 00000100:00100000:0.0:1767319453.405015:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2211 00000100:00100000:0.0:1767319453.405020:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985173760:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319453.405035:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319453.405069:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985173760:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (140us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319453.405076:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2211 00000100:00080000:0.0:1767319453.405094:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319453.405098:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319453.405123:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985173760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319453.405244:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985173632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 553us (676us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319453.405251:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2210 00000100:00100000:0.0:1767319453.405474:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985173632:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319453.531432:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319454.556590:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319454.894030:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319454.894145:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319454.903954:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561664 00000100:00100000:2.0:1767319454.903967:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561664 00000100:00100000:2.0:1767319454.904068:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561536 00000100:00100000:2.0:1767319454.904070:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561536 00000100:00080000:3.0:1767319454.904102:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319454.908949:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4628 00000100:00100000:0.0:1767319454.908976:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920561664:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319454.909071:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920561664:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 111us (15053us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319454.909077:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4628 00000100:00100000:0.0:1767319454.909096:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5107 00000100:00100000:0.0:1767319454.909099:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920561536:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319454.909115:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319454.909134:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920561536:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 34us (14991us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319454.909138:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5107 00010000:00010000:2.0:1767319455.579421:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319455.963305:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319455.963316:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319455.963320:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319455.963324:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319455.963328:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319455.963336:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4699 00000100:00080000:3.0:1767319455.963341:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319455.963348:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319455.963399:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc700 x1853163985173888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319455.963428:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319455.963432:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319455.963434:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319455.963437:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4699 00000100:00080000:3.0:1767319455.963439:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319455.963443:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319455.963454:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffd180 x1853163985174016/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319455.963487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319455.963491:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319455.963492:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319455.963496:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4699 00000100:00080000:3.0:1767319455.963498:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319455.963501:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319455.963512:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffed80 x1853163985174272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319455.963533:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319455.963536:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319455.963538:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319455.963541:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4699 00000100:00080000:3.0:1767319455.963543:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319455.963546:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319455.963556:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc000 x1853163985174528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319455.963673:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985174144:0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.963741:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319455.963811:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319455.963818:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985174784:0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.963845:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319455.963880:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174784 00000100:00100000:0.0:1767319455.963888:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174784 00000100:00100000:0.0:1767319455.963937:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3045 00000100:00100000:0.0:1767319455.963954:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985174784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.964003:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985174784:12345-0@lo:400:kworker.0 Request processed in 58us (157us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319455.964011:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3045 00000100:00100000:0.0:1767319455.964042:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985174656:0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.964065:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319455.964082:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319455.964087:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985174400:0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.964107:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319455.964143:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985174784:0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.964194:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174656 00000100:00100000:0.0:1767319455.964197:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174656 00000100:00100000:0.0:1767319455.964217:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3046 00000100:00100000:0.0:1767319455.964222:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985174656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.964252:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985174656:12345-0@lo:400:kworker.0 Request processed in 29us (186us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319455.964259:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3046 00000100:00100000:0.0:1767319455.964299:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985174656:0@lo:400:kworker.0 00000100:00080000:3.0:1767319455.964494:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319455.964498:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319455.964509:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174144 00000100:00100000:3.0:1767319455.964513:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174144 00000100:00100000:3.0:1767319455.964550:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174400 00000100:00100000:3.0:1767319455.964553:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174400 00000100:00100000:3.0:1767319455.964575:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4629 00000100:00100000:3.0:1767319455.964581:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985174144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319455.964617:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985174144:12345-0@lo:400:kworker.0 Request processed in 36us (876us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319455.964623:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4629 00000100:00100000:3.0:1767319455.964634:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4630 00000100:00100000:3.0:1767319455.964640:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985174400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319455.964665:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985174400:12345-0@lo:400:kworker.0 Request processed in 26us (557us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319455.964671:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4630 00000100:00080000:0.0:1767319455.964815:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319455.964836:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985174144:0@lo:400:kworker.0 00000100:00100000:0.0:1767319455.964860:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985174400:0@lo:400:kworker.0 00000100:00100000:2.0:1767319455.964931:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985173888:0@lo:38:kworker.0 00000100:00100000:2.0:1767319455.964960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319455.964993:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985174016:0@lo:38:kworker.0 00000100:00100000:2.0:1767319455.965016:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319455.965048:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985174272:0@lo:38:kworker.0 00000100:00100000:2.0:1767319455.965069:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319455.965086:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985174528:0@lo:38:kworker.0 00000100:00100000:2.0:1767319455.965105:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319455.965132:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985173888 00000100:00100000:2.0:1767319455.965137:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985173888 00000100:00100000:2.0:1767319455.965157:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174272 00000100:00100000:2.0:1767319455.965159:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174272 00000100:00100000:2.0:1767319455.965191:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174528 00000100:00100000:2.0:1767319455.965194:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174528 00000100:00100000:2.0:1767319455.965203:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5108 00000100:00100000:2.0:1767319455.965207:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985173888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319455.965225:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319455.965256:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985173888:12345-0@lo:38:kworker.0 Request processed in 47us (296us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319455.965261:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5108 00000100:00100000:2.0:1767319455.965273:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5109 00000100:00100000:2.0:1767319455.965276:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985174272:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319455.965284:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319455.965703:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5110 00000100:00100000:3.0:1767319455.965708:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985174528:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319455.965717:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319455.965744:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985174528:12345-0@lo:38:kworker.0 Request processed in 36us (638us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319455.965749:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5110 00000100:00080000:3.0:1767319455.965801:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319455.965805:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319455.965808:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985173888:0@lo:38:kworker.0 00000100:00080000:3.0:1767319455.965814:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319455.965817:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319455.965820:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985174528:0@lo:38:kworker.0 00000100:00100000:0.0:1767319455.965886:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174016 00000100:00100000:0.0:1767319455.965890:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174016 00000100:00100000:0.0:1767319455.965919:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1767 00000100:00100000:0.0:1767319455.965923:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985174016:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319455.965932:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319455.965960:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985174016:12345-0@lo:38:kworker.0 Request processed in 35us (943us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319455.965965:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1767 00000100:00080000:3.0:1767319455.966058:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319455.966062:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319455.966065:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985174016:0@lo:38:kworker.0 00000100:00100000:2.0:1767319455.966254:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985174272:12345-0@lo:38:kworker.0 Request processed in 975us (1183us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319455.966258:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5109 00000100:00080000:3.0:1767319455.966416:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319455.966421:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319455.966424:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985174272:0@lo:38:kworker.0 00010000:00010000:1.0:1767319456.603709:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319457.627293:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319458.523863:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985174912:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319458.523938:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319458.524072:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985174912 00000100:00100000:0.0:1767319458.524082:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985174912 00000100:00100000:0.0:1767319458.524154:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2212 00000100:00100000:0.0:1767319458.524181:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985174912:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319458.524211:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319458.524257:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985174912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (318us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319458.524265:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2212 00000100:00100000:0.0:1767319458.524307:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985174912:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319458.524348:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319458.524502:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319458.524515:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985175040:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319458.524550:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319458.524932:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175040 00000100:00100000:2.0:1767319458.524938:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175040 00000100:00100000:2.0:1767319458.524987:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2213 00000100:00100000:2.0:1767319458.524995:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985175040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319458.525013:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319458.525054:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985175040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (502us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319458.525060:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2213 00000100:00100000:2.0:1767319458.525093:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985175040:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319458.525157:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767319458.651570:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319459.675450:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319460.010631:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319460.010729:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561920 00000100:00100000:1.0:1767319460.010740:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561920 00000100:00100000:1.0:1767319460.010818:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4631 00000100:00100000:1.0:1767319460.010830:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920561920:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319460.010908:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920561920:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (287us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319460.010926:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4631 00000100:00080000:0.0:1767319460.011050:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319460.011253:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319460.011376:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920561792 00000100:00100000:0.0:1767319460.011381:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920561792 00000100:00100000:0.0:1767319460.011410:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5111 00000100:00100000:0.0:1767319460.011416:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920561792:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319460.011436:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319460.011479:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920561792:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 61us (226us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319460.011486:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5111 00010000:00010000:1.0:1767319460.699401:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319461.084239:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319461.084249:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319461.084253:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319461.084257:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319461.084284:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319461.084293:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4704 00000100:00080000:0.0:1767319461.084296:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319461.084303:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319461.084344:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985175168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319461.084379:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319461.084383:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319461.084385:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319461.084388:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4704 00000100:00080000:0.0:1767319461.084391:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319461.084394:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319461.084408:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985175296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319461.084451:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319461.084454:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319461.084456:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319461.084460:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4704 00000100:00080000:0.0:1767319461.084462:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319461.084465:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319461.084479:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba4e00 x1853163985175552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319461.084504:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319461.084507:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319461.084509:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319461.084513:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4704 00000100:00080000:0.0:1767319461.084515:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319461.084518:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319461.084526:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba6a00 x1853163985175808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319461.084575:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985175936:0@lo:400:kworker.0 00000100:00100000:0.0:1767319461.084846:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319461.084906:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319461.084913:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985175680:0@lo:400:kworker.0 00000100:00100000:0.0:1767319461.084937:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319461.085104:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985176064:0@lo:400:kworker.0 00000100:00100000:3.0:1767319461.085133:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319461.085183:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319461.085190:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985175424:0@lo:400:kworker.0 00000100:00100000:3.0:1767319461.085212:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319461.085253:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175680 00000100:00100000:3.0:1767319461.085273:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175680 00000100:00100000:3.0:1767319461.085323:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175424 00000100:00100000:3.0:1767319461.085327:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175424 00000100:00100000:3.0:1767319461.085348:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4632 00000100:00100000:3.0:1767319461.085356:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985175680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319461.085406:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985175680:12345-0@lo:400:kworker.0 Request processed in 50us (467us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319461.085413:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4632 00000100:00100000:3.0:1767319461.085437:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4633 00000100:00100000:3.0:1767319461.085443:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985175424:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319461.085482:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985175424:12345-0@lo:400:kworker.0 Request processed in 39us (269us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319461.085487:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4633 00000100:00080000:3.0:1767319461.085507:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319461.085536:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985175424:0@lo:400:kworker.0 00000100:00100000:3.0:1767319461.085567:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985175680:0@lo:400:kworker.0 00000100:00100000:2.0:1767319461.085640:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985175168:0@lo:38:kworker.0 00000100:00100000:2.0:1767319461.085669:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319461.085694:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985175296:0@lo:38:kworker.0 00000100:00100000:2.0:1767319461.085716:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319461.085740:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985175552:0@lo:38:kworker.0 00000100:00100000:2.0:1767319461.085760:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319461.085775:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985175808:0@lo:38:kworker.0 00000100:00100000:2.0:1767319461.085793:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319461.085820:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175296 00000100:00100000:2.0:1767319461.085824:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175296 00000100:00100000:2.0:1767319461.085845:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1768 00000100:00100000:2.0:1767319461.085850:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985175296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319461.085867:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319461.085912:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985175296:12345-0@lo:38:kworker.0 Request processed in 48us (182us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319461.085917:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1768 00000100:00100000:2.0:1767319461.085936:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175168 00000100:00100000:2.0:1767319461.085939:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175168 00000100:00100000:2.0:1767319461.085953:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175552 00000100:00100000:2.0:1767319461.085956:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175552 00000100:00100000:2.0:1767319461.085967:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175808 00000100:00100000:2.0:1767319461.085969:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175808 00000100:00100000:2.0:1767319461.085978:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5112 00000100:00100000:2.0:1767319461.085982:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985175168:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319461.085997:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319461.086022:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985175168:12345-0@lo:38:kworker.0 Request processed in 38us (352us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319461.086028:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5112 00000100:00100000:2.0:1767319461.086039:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5113 00000100:00100000:2.0:1767319461.086042:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985175552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319461.086051:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319461.086072:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985175552:12345-0@lo:38:kworker.0 Request processed in 28us (311us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319461.086076:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5113 00000100:00100000:2.0:1767319461.086087:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5114 00000100:00100000:2.0:1767319461.086090:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985175808:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319461.086098:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319461.086119:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985175936 00000100:00100000:2.0:1767319461.086119:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985175808:12345-0@lo:38:kworker.0 Request processed in 27us (325us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319461.086124:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985175936 00000100:00100000:2.0:1767319461.086125:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5114 00000100:00080000:2.0:1767319461.086153:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319461.086158:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319461.086162:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985175168:0@lo:38:kworker.0 00000100:00100000:1.0:1767319461.086190:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985176064 00000100:00080000:2.0:1767319461.086190:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319461.086194:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985176064 00000100:00080000:2.0:1767319461.086194:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319461.086197:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985175296:0@lo:38:kworker.0 00000100:00080000:2.0:1767319461.086203:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319461.086206:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319461.086209:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985175552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319461.086217:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3047 00000100:00080000:2.0:1767319461.086217:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319461.086220:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319461.086223:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985175936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319461.086223:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985175808:0@lo:38:kworker.0 00000100:00100000:1.0:1767319461.086276:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985175936:12345-0@lo:400:kworker.0 Request processed in 38us (1416us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319461.086282:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3047 00000100:00080000:2.0:1767319461.086288:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319461.086290:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319461.086295:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3048 00000100:00100000:1.0:1767319461.086301:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985176064:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319461.086325:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985175936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319461.086331:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985176064:12345-0@lo:400:kworker.0 Request processed in 31us (1199us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319461.086336:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3048 00000100:00100000:1.0:1767319461.086359:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985176064:0@lo:400:kworker.0 00010000:00010000:2.0:1767319461.723366:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319462.747416:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319463.643482:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985176192:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319463.643594:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319463.643719:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985176192 00000100:00100000:3.0:1767319463.643729:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985176192 00000100:00100000:3.0:1767319463.643867:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2214 00000100:00100000:3.0:1767319463.643879:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985176192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319463.643918:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319463.643988:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985176192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 111us (392us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319463.643996:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2214 00000100:00100000:3.0:1767319463.644058:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985176192:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319463.644429:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985176320:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319463.644469:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319463.644686:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319463.644691:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985176320 00000100:00100000:0.0:1767319463.644699:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985176320 00000100:00080000:2.0:1767319463.644729:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319463.644748:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2215 00000100:00100000:0.0:1767319463.644755:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985176320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319463.644772:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319463.644809:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985176320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 54us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319463.644817:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2215 00000100:00100000:1.0:1767319463.644848:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985176320:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319463.771460:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319464.795632:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319465.129902:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920562048 offset 0 length 520: 4 00000100:00100000:3.0:1767319465.134665:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319465.134827:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920562176 00000100:00100000:0.0:1767319465.134838:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920562176 00000100:00100000:0.0:1767319465.134924:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5115 00000100:00100000:0.0:1767319465.134932:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920562176:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319465.134955:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319465.135021:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920562176:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 87us (365us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319465.135029:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5115 00000100:00100000:3.0:1767319465.139815:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319465.140041:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920562304 00000100:00100000:0.0:1767319465.140057:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920562304 00000100:00080000:2.0:1767319465.140291:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319465.140308:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4634 00000100:00100000:3.0:1767319465.140328:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920562304:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319465.140456:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920562304:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 136us (650us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319465.140472:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4634 00000100:00100000:2.0:1767319465.179600:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319440/real 1767319440] req@ffff8ce1b5bebb80 x1853163985170944/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319465 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319465.179645:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5bebb80 x1853163985170944/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319465 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319465.179670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319465.179705:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319465.179709:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985170944:0@lo:250:kworker.0 00000100:00080000:0.0:1767319465.179805:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319465.179813:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319465.179816:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319465.179821:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4688 00000100:00080000:0.0:1767319465.179824:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319465.179832:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319465.180489:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba4a80 x1853163985176448/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319465.180521:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319465.180526:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319465.180528:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319465.180532:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4709 00000100:00080000:0.0:1767319465.180535:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319465.180540:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319465.180562:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba6300 x1853163985176576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319465.180569:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319465.180571:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319465.180573:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319465.180575:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4709 00000100:00080000:0.0:1767319465.180577:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319465.180579:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319465.180586:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba6680 x1853163985176704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319465.180624:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319465.180629:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319465.180631:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319465.180635:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4709 00000100:00080000:0.0:1767319465.180637:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319465.180641:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319465.180652:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba5880 x1853163985176960/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319465.180662:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985176832:0@lo:400:kworker.0 00000100:00080000:0.0:1767319465.180678:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319465.180681:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319465.180683:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319465.180686:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4709 00000100:00080000:0.0:1767319465.180689:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319465.180692:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319465.180705:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba5180 x1853163985177216/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319465.180750:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985177088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319465.180831:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319465.180832:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985176448:0@lo:250:kworker.0 00000100:00100000:3.0:1767319465.180846:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985177472:0@lo:400:kworker.0 00000400:00000100:2.0:1767319465.180860:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985176448 offset 0 length 520: 4 00000100:00100000:1.0:1767319465.180977:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985177344:0@lo:400:kworker.0 00000100:00100000:3.0:1767319465.182269:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319465.182287:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319465.182330:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177472 00000100:00100000:3.0:1767319465.182331:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985176832 00000100:00100000:0.0:1767319465.182340:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177472 00000100:00100000:1.0:1767319465.182377:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319465.182419:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985176832 00000100:00100000:2.0:1767319465.182458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985176576:0@lo:38:kworker.0 00000100:00100000:3.0:1767319465.182479:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177088 00000100:00100000:3.0:1767319465.182484:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177088 00000100:00100000:2.0:1767319465.182493:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319465.182500:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177344 00000100:00100000:1.0:1767319465.182506:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177344 00000100:00100000:2.0:1767319465.182512:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985176704:0@lo:38:kworker.0 00000100:00100000:3.0:1767319465.182525:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985176576 00000100:00100000:3.0:1767319465.182530:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985176576 00000100:00100000:2.0:1767319465.182540:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319465.182561:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3049 00000100:00100000:3.0:1767319465.182561:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4635 00000100:00100000:2.0:1767319465.182562:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985176960:0@lo:38:kworker.0 00000100:00100000:3.0:1767319465.182573:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985176832:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319465.182579:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985177344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319465.182586:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319465.182598:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985177216:0@lo:38:kworker.0 00000100:00100000:2.0:1767319465.182613:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319465.182641:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319465.182646:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319465.182648:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319465.182653:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985177344:12345-0@lo:400:kworker.0 Request processed in 78us (274us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319465.182660:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985176960 00000100:00100000:1.0:1767319465.182664:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3049 00000100:00100000:2.0:1767319465.182664:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985176960 00000100:00100000:2.0:1767319465.182678:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177216 00000100:00100000:2.0:1767319465.182680:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177216 00000100:00100000:2.0:1767319465.182693:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4636 00000100:00100000:2.0:1767319465.182704:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985177088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319465.182729:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985177344:0@lo:400:kworker.0 00000100:00100000:2.0:1767319465.182751:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985177088:12345-0@lo:400:kworker.0 Request processed in 49us (462us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319465.182758:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4636 00000100:00100000:2.0:1767319465.182776:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5116 00000100:00100000:2.0:1767319465.182780:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985176576:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319465.182786:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5117 00000100:00100000:1.0:1767319465.182793:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985176960:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319465.182801:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767319465.182807:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319465.182830:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985176576:12345-0@lo:38:kworker.0 Request processed in 49us (336us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319465.182834:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5116 00000100:00100000:2.0:1767319465.182843:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5118 00000100:00100000:2.0:1767319465.182847:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985177216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319465.182854:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319465.182862:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985176960:12345-0@lo:38:kworker.0 Request processed in 67us (277us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319465.182870:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5117 00000100:00100000:1.0:1767319465.182932:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985176704 00000100:00100000:1.0:1767319465.182937:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985176704 00000100:00100000:1.0:1767319465.182974:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1769 00000100:00100000:1.0:1767319465.182979:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985176704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319465.183005:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319465.183035:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985176704:12345-0@lo:38:kworker.0 Request processed in 54us (494us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319465.183041:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1769 00000100:00100000:0.0:1767319465.183099:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3050 00000100:00100000:0.0:1767319465.183107:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985177472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319465.183148:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985177472:12345-0@lo:400:kworker.0 Request processed in 42us (884us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319465.183155:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3050 00000100:00100000:3.0:1767319465.183181:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985176832:12345-0@lo:400:kworker.0 Request processed in 591us (2331us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319465.183201:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4635 00000100:00080000:1.0:1767319465.183240:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319465.183250:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985177088:0@lo:400:kworker.0 00000100:00080000:1.0:1767319465.183250:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319465.183260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985176576:0@lo:38:kworker.0 00000100:00100000:3.0:1767319465.183260:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985176832:0@lo:400:kworker.0 00000100:00080000:1.0:1767319465.183282:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319465.183310:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985177472:0@lo:400:kworker.0 00000100:00080000:1.0:1767319465.183627:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319465.183634:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985176704:0@lo:38:kworker.0 00000100:00080000:1.0:1767319465.183653:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319465.183658:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319465.183661:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985176960:0@lo:38:kworker.0 00000100:00080000:1.0:1767319465.183689:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319465.183695:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319465.183695:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985177216:12345-0@lo:38:kworker.0 Request processed in 844us (1078us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319465.183700:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985177216:0@lo:38:kworker.0 00000100:00100000:2.0:1767319465.183710:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5118 00010000:00010000:1.0:1767319465.820119:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319466.843419:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319467.867477:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319468.763476:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319468.763506:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985177728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319468.763508:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985177600:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319468.763601:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319468.763837:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319468.764064:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177600 00000100:00100000:0.0:1767319468.764075:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177600 00000100:00100000:0.0:1767319468.764153:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177728 00000100:00100000:0.0:1767319468.764156:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177728 00000100:00100000:0.0:1767319468.764213:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2216 00000100:00100000:0.0:1767319468.764222:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985177600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319468.764259:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319468.764308:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985177600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (708us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319468.764316:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2216 00000100:00100000:0.0:1767319468.764337:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2217 00000100:00100000:0.0:1767319468.764343:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985177728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319468.764359:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319468.764404:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985177600:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319468.764416:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319468.764422:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319468.764910:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985177728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 566us (1072us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319468.764917:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2217 00000100:00100000:1.0:1767319468.765863:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985177728:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319468.895382:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319469.917438:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319470.263025:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319470.263072:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319470.263570:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920562560 00000100:00100000:1.0:1767319470.265689:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920562560 00000100:00100000:0.0:1767319470.265946:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920562432 00000100:00080000:2.0:1767319470.265955:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319470.265959:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920562432 00000100:00100000:1.0:1767319470.266030:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4637 00000100:00100000:1.0:1767319470.266049:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920562560:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319470.266066:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5119 00000100:00100000:0.0:1767319470.266075:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920562432:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319470.266114:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319470.266151:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920562560:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (3132us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319470.266161:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4637 00000100:00100000:0.0:1767319470.266213:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920562432:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 136us (3138us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319470.266220:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5119 00000100:00080000:2.0:1767319470.305879:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319470.305894:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319470.305899:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319470.305905:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319470.305910:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319470.305923:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4713 00000100:00080000:2.0:1767319470.305929:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319470.305938:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319470.306001:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2680 x1853163985177856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319470.306031:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319470.306036:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319470.306039:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319470.306045:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4713 00000100:00080000:2.0:1767319470.306049:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319470.306054:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319470.306075:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1f80 x1853163985177984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319470.306153:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319470.306158:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319470.306161:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319470.306192:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4713 00000100:00080000:2.0:1767319470.306196:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319470.306201:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319470.306220:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0e00 x1853163985178240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319470.306266:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319470.306271:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319470.306273:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319470.306280:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4713 00000100:00080000:2.0:1767319470.306283:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319470.306287:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319470.306314:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3100 x1853163985178496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319470.306398:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985178112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.306490:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.306598:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319470.306609:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985178752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.306652:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.306713:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985178624:0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.306749:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.306783:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319470.306790:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985178368:0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.306817:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.306860:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985178112 00000100:00100000:2.0:1767319470.306868:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985178112 00000100:00100000:2.0:1767319470.306937:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985178368 00000100:00100000:2.0:1767319470.306941:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985178368 00000100:00100000:2.0:1767319470.306965:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4638 00000100:00100000:2.0:1767319470.306975:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985178112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.307032:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985178112:12345-0@lo:400:kworker.0 Request processed in 58us (546us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319470.307042:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4638 00000100:00100000:2.0:1767319470.307062:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4639 00000100:00100000:2.0:1767319470.307068:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985178368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.307108:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985178368:12345-0@lo:400:kworker.0 Request processed in 31us (280us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319470.307114:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4639 00000100:00100000:2.0:1767319470.307185:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985178112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.307217:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985178368:0@lo:400:kworker.0 00000100:00080000:2.0:1767319470.307233:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319470.307294:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985178752 00000100:00100000:2.0:1767319470.307298:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985178752 00000100:00100000:2.0:1767319470.307329:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985178624 00000100:00100000:2.0:1767319470.307332:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985178624 00000100:00100000:2.0:1767319470.307354:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3051 00000100:00100000:2.0:1767319470.307361:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985178752:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.307398:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985178752:12345-0@lo:400:kworker.0 Request processed in 38us (749us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319470.307404:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3051 00000100:00100000:2.0:1767319470.307415:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3052 00000100:00100000:2.0:1767319470.309212:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985178624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.309707:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985178624:12345-0@lo:400:kworker.0 Request processed in 2284us (2961us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319470.309719:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3052 00000100:00100000:2.0:1767319470.309786:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985177856:0@lo:38:kworker.0 00000100:00100000:2.0:1767319470.309837:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.309870:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985177984:0@lo:38:kworker.0 00000100:00100000:2.0:1767319470.309897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.309934:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985178240:0@lo:38:kworker.0 00000100:00100000:2.0:1767319470.309956:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.309973:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985178496:0@lo:38:kworker.0 00000100:00100000:2.0:1767319470.309995:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319470.310058:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985178752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.310094:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985178624:0@lo:400:kworker.0 00000100:00100000:2.0:1767319470.310128:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177856 00000100:00100000:2.0:1767319470.310135:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177856 00000100:00100000:2.0:1767319470.310192:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985178240 00000100:00100000:2.0:1767319470.310195:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985178240 00000100:00100000:2.0:1767319470.310212:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985178496 00000100:00100000:2.0:1767319470.310215:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985178496 00000100:00100000:2.0:1767319470.310227:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5120 00000100:00100000:2.0:1767319470.310234:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985177856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319470.310258:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319470.310304:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985177856:12345-0@lo:38:kworker.0 Request processed in 62us (461us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319470.310311:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5120 00000100:00100000:2.0:1767319470.310339:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5121 00000100:00100000:2.0:1767319470.310344:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985178240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319470.310353:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319470.310378:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985178240:12345-0@lo:38:kworker.0 Request processed in 33us (421us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319470.310385:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5121 00000100:00100000:2.0:1767319470.310396:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5122 00000100:00100000:2.0:1767319470.310400:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985178496:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319470.310409:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319470.310431:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985178496:12345-0@lo:38:kworker.0 Request processed in 29us (436us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319470.310436:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5122 00000100:00080000:2.0:1767319470.310467:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319470.310473:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319470.310476:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985177856:0@lo:38:kworker.0 00000100:00080000:2.0:1767319470.310484:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319470.310488:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319470.310491:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985178240:0@lo:38:kworker.0 00000100:00080000:2.0:1767319470.310498:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319470.310501:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319470.310504:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985178496:0@lo:38:kworker.0 00000100:00080000:2.0:1767319470.310526:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319470.310531:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319470.314259:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985177984 00000100:00100000:1.0:1767319470.314295:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985177984 00000100:00100000:1.0:1767319470.314377:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1770 00000100:00100000:1.0:1767319470.314791:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985177984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319470.314817:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319470.314904:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985177984:12345-0@lo:38:kworker.0 Request processed in 115us (5005us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319470.314911:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1770 00000100:00080000:2.0:1767319470.316439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319470.316457:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319470.316463:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985177984:0@lo:38:kworker.0 00010000:00010000:0.0:1767319470.939453:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319471.966388:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319472.988610:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319473.883423:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319473.883455:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985179008:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319473.883545:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319473.883720:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179008 00000100:00100000:0.0:1767319473.883730:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179008 00000100:00080000:3.0:1767319473.884002:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319473.884191:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2218 00000100:00100000:3.0:1767319473.884205:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985179008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319473.884251:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319473.884319:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985179008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 118us (774us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319473.884328:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2218 00000100:00100000:3.0:1767319473.884388:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985179008:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319473.884617:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985178880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319473.884657:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319473.884759:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985178880 00000100:00100000:0.0:1767319473.884766:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985178880 00000100:00100000:0.0:1767319473.884980:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2219 00000100:00100000:0.0:1767319473.884990:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985178880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319473.885010:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319473.885157:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985178880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 169us (499us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319473.885185:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2219 00000100:00080000:3.0:1767319473.885247:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319473.885318:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985178880:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319474.014351:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319475.042913:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319475.375395:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319475.375456:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319475.375654:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920562688 00000100:00100000:2.0:1767319475.375669:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920562688 00000100:00100000:2.0:1767319475.375730:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920562816 00000100:00100000:2.0:1767319475.375734:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920562816 00000100:00100000:2.0:1767319475.375806:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4640 00000100:00100000:2.0:1767319475.375820:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920562816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319475.375897:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5123 00000100:00100000:3.0:1767319475.375913:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920562688:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:2.0:1767319475.375927:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920562816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 109us (467us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319475.375937:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4640 00010000:00020000:3.0:1767319475.375942:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319475.376019:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920562688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (628us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319475.376030:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5123 00000100:00080000:1.0:1767319475.376372:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319475.419633:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319475.419661:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319475.419665:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319475.419670:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319475.419675:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319475.419686:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4718 00000100:00080000:1.0:1767319475.419691:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319475.419700:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319475.419755:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5f80 x1853163985179136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319475.419785:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319475.420128:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319475.420131:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319475.420136:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4718 00000100:00080000:1.0:1767319475.420138:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319475.420144:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319475.420162:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7100 x1853163985179264/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319475.420333:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319475.420337:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319475.420339:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319475.420343:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4718 00000100:00080000:1.0:1767319475.420345:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319475.420349:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319475.420362:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9a300 x1853163985179520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319475.420392:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319475.420396:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319475.420398:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319475.420401:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4718 00000100:00080000:1.0:1767319475.420403:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319475.420407:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319475.420419:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98700 x1853163985179776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319475.420463:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985179648:0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.420542:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319475.420623:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319475.420637:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985179904:0@lo:400:kworker.0 00000100:00100000:3.0:1767319475.420643:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985179392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.420672:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319475.420755:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985179136:0@lo:38:kworker.0 00000100:00100000:2.0:1767319475.421051:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179904 00000100:00100000:2.0:1767319475.421061:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179904 00000100:00100000:2.0:1767319475.421134:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3053 00000100:00100000:2.0:1767319475.421146:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985179904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.430102:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179648 00000100:00100000:1.0:1767319475.430125:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179648 00000100:00100000:1.0:1767319475.430234:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4641 00000100:00100000:1.0:1767319475.430248:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985179648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.430358:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985179648:12345-0@lo:400:kworker.0 Request processed in 112us (9815us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319475.430366:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4641 00000100:00080000:1.0:1767319475.430395:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319475.430401:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319475.430448:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985179648:0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.430523:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985180032:0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.430568:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319475.430611:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180032 00000100:00100000:1.0:1767319475.430616:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180032 00000100:00100000:1.0:1767319475.430659:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3054 00000100:00100000:1.0:1767319475.430667:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985180032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.430705:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985180032:12345-0@lo:400:kworker.0 Request processed in 40us (138us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319475.430712:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3054 00000100:00080000:1.0:1767319475.430727:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319475.430744:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985180032:0@lo:400:kworker.0 00000100:00100000:3.0:1767319475.431965:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319475.432424:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319475.432474:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985179264:0@lo:38:kworker.0 00000100:00100000:0.0:1767319475.432518:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319475.432549:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985179520:0@lo:38:kworker.0 00000100:00100000:0.0:1767319475.432572:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319475.432592:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985179776:0@lo:38:kworker.0 00000100:00100000:0.0:1767319475.432616:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319475.432662:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179264 00000100:00100000:0.0:1767319475.432670:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179264 00000100:00100000:0.0:1767319475.432708:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1771 00000100:00100000:0.0:1767319475.432716:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985179264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319475.432742:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319475.432903:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179392 00000100:00100000:1.0:1767319475.432907:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179392 00000100:00100000:1.0:1767319475.432958:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179136 00000100:00100000:1.0:1767319475.432961:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179136 00000100:00100000:1.0:1767319475.432975:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179520 00000100:00100000:1.0:1767319475.432978:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179520 00000100:00100000:1.0:1767319475.432986:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985179776 00000100:00100000:1.0:1767319475.432988:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985179776 00000100:00100000:1.0:1767319475.432998:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4642 00000100:00100000:1.0:1767319475.433017:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985179392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.433082:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985179392:12345-0@lo:400:kworker.0 Request processed in 78us (1124us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319475.433096:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5124 00000100:00100000:0.0:1767319475.433104:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985179264:12345-0@lo:38:kworker.0 Request processed in 71us (268us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319475.433105:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985179136:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319475.433128:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1771 00010000:00020000:3.0:1767319475.433137:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319475.433210:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985179136:12345-0@lo:38:kworker.0 Request processed in 106us (791us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319475.433253:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4642 00000100:00100000:1.0:1767319475.433272:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5125 00000100:00100000:1.0:1767319475.433277:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985179520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319475.433288:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319475.433377:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985179520:12345-0@lo:38:kworker.0 Request processed in 98us (803us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319475.433384:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5125 00000100:00100000:1.0:1767319475.433395:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5126 00000100:00100000:1.0:1767319475.433399:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985179776:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319475.433409:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319475.438796:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5124 00000100:00080000:0.0:1767319475.438885:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319475.438899:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319475.438909:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985179136:0@lo:38:kworker.0 00000100:00100000:3.0:1767319475.438911:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985179392:0@lo:400:kworker.0 00000100:00080000:0.0:1767319475.438935:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319475.438942:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319475.438946:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985179264:0@lo:38:kworker.0 00000100:00080000:0.0:1767319475.438961:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319475.438965:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319475.438969:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985179520:0@lo:38:kworker.0 00000100:00080000:3.0:1767319475.438981:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319475.439234:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985179904:12345-0@lo:400:kworker.0 Request processed in 18087us (18558us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319475.439248:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3053 00000100:00100000:0.0:1767319475.439701:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985179904:0@lo:400:kworker.0 00000100:00100000:1.0:1767319475.439947:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985179776:12345-0@lo:38:kworker.0 Request processed in 6544us (7327us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319475.439958:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5126 00000100:00080000:0.0:1767319475.440024:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319475.440029:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319475.440033:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985179776:0@lo:38:kworker.0 00010000:00010000:1.0:1767319476.059393:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319477.083391:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319478.107645:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319479.004245:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319479.004268:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985180160:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319479.004349:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319479.004517:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180160 00000100:00100000:3.0:1767319479.004525:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180160 00000100:00100000:3.0:1767319479.004598:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2220 00000100:00100000:3.0:1767319479.004609:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985180160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319479.004645:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319479.004711:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985180160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319479.004719:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2220 00000100:00100000:3.0:1767319479.004768:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985180160:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319479.004981:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319479.006652:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985180288:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319479.006808:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319479.007338:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180288 00000100:00100000:3.0:1767319479.007349:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180288 00000100:00100000:3.0:1767319479.007417:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2221 00000100:00100000:3.0:1767319479.007426:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985180288:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319479.007456:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767319479.007526:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319479.011645:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985180288:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4209us (4826us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319479.011662:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2221 00000100:00100000:0.0:1767319479.014106:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985180288:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319479.135491:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319480.158808:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319480.492093:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319480.492144:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319480.492303:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920563072 00000100:00100000:2.0:1767319480.492324:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920563072 00000100:00100000:2.0:1767319480.492393:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920562944 00000100:00100000:2.0:1767319480.492397:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920562944 00000100:00100000:2.0:1767319480.492415:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4643 00000100:00100000:2.0:1767319480.492427:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920563072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319480.492509:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920563072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (428us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319480.492519:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4643 00000100:00100000:2.0:1767319480.492538:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5127 00000100:00100000:2.0:1767319480.492544:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920562944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319480.492563:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319480.492587:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920562944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (442us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319480.492593:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5127 00000100:00080000:1.0:1767319480.492807:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319480.539333:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319480.539346:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319480.539350:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319480.539355:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319480.539359:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319480.539369:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4723 00000100:00080000:1.0:1767319480.539375:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319480.539381:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319480.539425:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98e00 x1853163985180416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319480.539453:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319480.539457:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319480.539460:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319480.539463:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4723 00000100:00080000:1.0:1767319480.539466:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319480.539470:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319480.539483:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9b800 x1853163985180544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319480.539535:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319480.539539:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319480.539541:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319480.539545:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4723 00000100:00080000:1.0:1767319480.539547:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319480.539551:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319480.539562:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99180 x1853163985180800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319480.539595:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319480.539599:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319480.539601:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319480.539604:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4723 00000100:00080000:1.0:1767319480.539607:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319480.539611:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319480.539622:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98000 x1853163985181056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319480.539668:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985181184:0@lo:400:kworker.0 00000100:00100000:1.0:1767319480.539756:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319480.539819:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319480.539826:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985180928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319480.539853:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.539994:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985180672:0@lo:400:kworker.0 00000100:00100000:3.0:1767319480.540023:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.540056:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319480.540061:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985181312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319480.540082:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.540116:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181184 00000100:00100000:3.0:1767319480.540124:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181184 00000100:00100000:3.0:1767319480.540192:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181312 00000100:00100000:3.0:1767319480.540196:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181312 00000100:00100000:3.0:1767319480.540221:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3055 00000100:00100000:3.0:1767319480.540230:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985181184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319480.540279:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985181184:12345-0@lo:400:kworker.0 Request processed in 50us (524us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319480.540290:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3055 00000100:00100000:3.0:1767319480.540306:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3056 00000100:00100000:3.0:1767319480.540312:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985181312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319480.540342:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985181312:12345-0@lo:400:kworker.0 Request processed in 30us (258us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319480.540347:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3056 00000100:00100000:3.0:1767319480.540389:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985181312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319480.540422:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985181184:0@lo:400:kworker.0 00000100:00100000:0.0:1767319480.540589:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180928 00000100:00100000:0.0:1767319480.540595:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180928 00000100:00100000:0.0:1767319480.540641:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180672 00000100:00100000:0.0:1767319480.540644:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180672 00000100:00100000:0.0:1767319480.540665:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4644 00000100:00100000:0.0:1767319480.540672:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985180928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319480.540712:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985180928:12345-0@lo:400:kworker.0 Request processed in 40us (857us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319480.540722:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4644 00000100:00100000:0.0:1767319480.540737:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4645 00000100:00100000:0.0:1767319480.540743:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985180672:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319480.540770:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985180672:12345-0@lo:400:kworker.0 Request processed in 28us (746us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319480.540776:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4645 00000100:00080000:3.0:1767319480.540845:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319480.540866:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985180672:0@lo:400:kworker.0 00000100:00100000:3.0:1767319480.540885:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985180928:0@lo:400:kworker.0 00000100:00100000:2.0:1767319480.541140:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985180416:0@lo:38:kworker.0 00000100:00100000:2.0:1767319480.541197:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319480.541373:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985180544:0@lo:38:kworker.0 00000100:00100000:2.0:1767319480.541399:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319480.541429:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985180800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319480.541450:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319480.542474:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180544 00000100:00100000:0.0:1767319480.542484:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180544 00000100:00100000:0.0:1767319480.542522:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1772 00000100:00100000:0.0:1767319480.542530:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985180544:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319480.542550:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319480.542597:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985180544:12345-0@lo:38:kworker.0 Request processed in 66us (1197us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319480.542604:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1772 00000100:00100000:3.0:1767319480.546134:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180416 00000100:00100000:3.0:1767319480.546155:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180416 00000100:00100000:3.0:1767319480.546244:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985180800 00000100:00100000:3.0:1767319480.546249:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985180800 00000100:00100000:3.0:1767319480.546275:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5128 00000100:00100000:3.0:1767319480.546284:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985180416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319480.546320:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319480.546408:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985180416:12345-0@lo:38:kworker.0 Request processed in 123us (5209us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319480.546415:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5128 00000100:00100000:3.0:1767319480.546433:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5129 00000100:00100000:3.0:1767319480.546437:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985180800:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319480.546447:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319480.546471:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985180800:12345-0@lo:38:kworker.0 Request processed in 32us (5021us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319480.546477:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5129 00000100:00080000:2.0:1767319480.547712:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319480.547723:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319480.547727:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985180800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319480.547740:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985181056:0@lo:38:kworker.0 00000100:00100000:2.0:1767319480.547787:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319480.547833:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319480.547836:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319480.547839:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985180416:0@lo:38:kworker.0 00000100:00080000:2.0:1767319480.547846:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319480.547848:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319480.547850:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985180544:0@lo:38:kworker.0 00000100:00100000:1.0:1767319480.548113:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181056 00000100:00100000:1.0:1767319480.548118:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181056 00000100:00100000:1.0:1767319480.548151:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5130 00000100:00100000:1.0:1767319480.548157:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985181056:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319480.548186:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319480.548220:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985181056:12345-0@lo:38:kworker.0 Request processed in 64us (434us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319480.548225:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5130 00000100:00080000:1.0:1767319480.548243:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319480.548245:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319480.548247:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985181056:0@lo:38:kworker.0 00010000:00010000:1.0:1767319481.180855:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319482.206125:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319483.227448:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319484.123324:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319484.123346:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985181440:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319484.123424:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319484.123834:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181440 00000100:00100000:2.0:1767319484.123852:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181440 00000100:00100000:2.0:1767319484.123920:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2222 00000100:00100000:2.0:1767319484.123931:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985181440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319484.123960:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319484.124018:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985181440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (592us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319484.124026:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2222 00000100:00100000:3.0:1767319484.124556:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985181440:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319484.125548:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319484.129151:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985181568:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319484.129249:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319484.129407:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181568 00000100:00100000:3.0:1767319484.129415:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181568 00000100:00100000:3.0:1767319484.129482:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2223 00000100:00100000:3.0:1767319484.129490:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985181568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319484.129518:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767319484.129587:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319484.133069:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985181568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3569us (3810us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319484.133085:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2223 00000100:00100000:2.0:1767319484.133349:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985181568:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319484.251449:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319485.278931:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319485.612476:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319485.612593:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319485.612963:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920563328 00000100:00100000:3.0:1767319485.612974:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920563328 00000100:00100000:3.0:1767319485.613054:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920563200 00000100:00100000:3.0:1767319485.613058:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920563200 00000100:00100000:3.0:1767319485.613086:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4646 00000100:00100000:3.0:1767319485.613097:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920563328:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319485.613211:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5131 00000100:00100000:2.0:1767319485.613218:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920563200:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319485.613239:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319485.613299:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920563200:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 80us (706us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319485.613307:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5131 00000100:00080000:0.0:1767319485.613401:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319485.614070:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920563328:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 972us (1603us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319485.614079:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4646 00000100:00080000:3.0:1767319485.659883:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319485.659895:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319485.659898:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319485.659902:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319485.659908:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319485.659918:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4728 00000100:00080000:3.0:1767319485.659922:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319485.659929:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319485.659973:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985181696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319485.660043:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985181696:0@lo:38:kworker.0 00000100:00100000:1.0:1767319485.660127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319485.660223:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181696 00000100:00100000:0.0:1767319485.660232:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181696 00000100:00100000:0.0:1767319485.660279:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5132 00000100:00100000:0.0:1767319485.660286:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985181696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319485.660308:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319485.660327:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319485.660332:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319485.660335:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319485.660339:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4728 00000100:00080000:3.0:1767319485.660343:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319485.660347:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319485.660364:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985181824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319485.660995:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985181824:0@lo:38:kworker.0 00000100:00100000:1.0:1767319485.661031:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319485.663554:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985181696:12345-0@lo:38:kworker.0 Request processed in 3257us (3417us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319485.663570:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5132 00000100:00080000:3.0:1767319485.664957:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319485.664963:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319485.664967:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319485.664974:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4728 00000100:00080000:3.0:1767319485.664979:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319485.664985:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319485.665012:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca5f80 x1853163985182080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319485.665039:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319485.665042:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319485.665044:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319485.665048:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4728 00000100:00080000:3.0:1767319485.665050:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319485.665054:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319485.665062:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7800 x1853163985182336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319485.665127:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985181952:0@lo:400:kworker.0 00000100:00100000:0.0:1767319485.665206:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319485.665288:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181824 00000100:00100000:2.0:1767319485.665298:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181824 00000100:00100000:2.0:1767319485.665335:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1773 00000100:00100000:2.0:1767319485.665343:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985181824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319485.665363:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319485.665401:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985181824:12345-0@lo:38:kworker.0 Request processed in 57us (4368us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319485.665408:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1773 00000100:00100000:2.0:1767319485.665445:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985182208:0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.665469:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319485.665521:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985182592:0@lo:400:kworker.0 00000100:00100000:3.0:1767319485.665550:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319485.665641:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319485.665647:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319485.665651:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985181824:0@lo:38:kworker.0 00000100:00080000:1.0:1767319485.665661:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319485.665664:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319485.665667:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985181696:0@lo:38:kworker.0 00000100:00100000:1.0:1767319485.665692:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985182080:0@lo:38:kworker.0 00000100:00100000:1.0:1767319485.665715:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319485.665733:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985182336:0@lo:38:kworker.0 00000100:00100000:1.0:1767319485.665750:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319485.665778:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985181952 00000100:00100000:1.0:1767319485.665782:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985181952 00000100:00100000:1.0:1767319485.665845:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182208 00000100:00100000:1.0:1767319485.665848:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182208 00000100:00100000:1.0:1767319485.665869:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182080 00000100:00100000:1.0:1767319485.665873:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182080 00000100:00100000:1.0:1767319485.665880:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182336 00000100:00100000:1.0:1767319485.665882:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182336 00000100:00100000:1.0:1767319485.665891:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4647 00000100:00100000:1.0:1767319485.665896:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985181952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319485.665932:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985181952:12345-0@lo:400:kworker.0 Request processed in 36us (727us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319485.665938:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4647 00000100:00100000:1.0:1767319485.665950:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4648 00000100:00100000:1.0:1767319485.665956:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985182208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319485.665981:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985182208:12345-0@lo:400:kworker.0 Request processed in 25us (510us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319485.665986:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4648 00000100:00100000:1.0:1767319485.665996:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5133 00000100:00100000:1.0:1767319485.666000:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985182080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319485.666011:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319485.666035:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985182080:12345-0@lo:38:kworker.0 Request processed in 33us (319us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319485.666040:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5133 00000100:00100000:1.0:1767319485.666050:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5134 00000100:00100000:1.0:1767319485.666054:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985182336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319485.666062:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319485.666081:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985182336:12345-0@lo:38:kworker.0 Request processed in 27us (330us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319485.666087:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5134 00000100:00100000:0.0:1767319485.667159:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985181952:0@lo:400:kworker.0 00000100:00080000:0.0:1767319485.667203:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319485.667316:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985182208:0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.667331:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319485.667339:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985182464:0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.667383:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319485.667445:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182592 00000100:00100000:2.0:1767319485.667451:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182592 00000100:00100000:2.0:1767319485.667485:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182464 00000100:00100000:2.0:1767319485.667488:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182464 00000100:00100000:2.0:1767319485.667505:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3057 00000100:00100000:2.0:1767319485.667513:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985182592:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.667550:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985182592:12345-0@lo:400:kworker.0 Request processed in 38us (1998us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319485.667556:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3057 00000100:00100000:2.0:1767319485.667571:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3058 00000100:00100000:2.0:1767319485.667578:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985182464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.667606:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985182464:12345-0@lo:400:kworker.0 Request processed in 29us (223us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319485.667615:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3058 00000100:00100000:2.0:1767319485.667644:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985182464:0@lo:400:kworker.0 00000100:00100000:3.0:1767319485.670200:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985182592:0@lo:400:kworker.0 00000100:00080000:1.0:1767319485.675342:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319485.675355:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319485.675360:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985182080:0@lo:38:kworker.0 00000100:00080000:1.0:1767319485.675372:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319485.675376:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319485.675379:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985182336:0@lo:38:kworker.0 00010000:00010000:2.0:1767319486.300592:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319487.323964:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319488.347408:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319489.245484:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985182720:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319489.245571:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319489.246205:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182720 00000100:00100000:2.0:1767319489.246217:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182720 00000100:00100000:2.0:1767319489.246298:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2224 00000100:00100000:2.0:1767319489.246310:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985182720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319489.246348:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767319489.246482:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319489.247108:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985182720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 800us (1534us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319489.247119:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2224 00000100:00100000:2.0:1767319489.247145:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319489.247160:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985182848:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319489.247219:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319489.247611:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182848 00000100:00100000:0.0:1767319489.247617:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182848 00000100:00100000:0.0:1767319489.247667:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2225 00000100:00100000:0.0:1767319489.247676:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985182848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319489.247695:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319489.247736:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985182848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (517us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319489.247742:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2225 00000100:00080000:2.0:1767319489.249200:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319489.249256:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985182848:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319489.257466:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985182720:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319489.371950:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767319489.710934:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920563456 offset 0 length 520: 4 00000100:00100000:0.0:1767319489.711914:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319489.712042:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920563584 00000100:00100000:2.0:1767319489.712053:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920563584 00000100:00100000:2.0:1767319489.712100:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5135 00000100:00100000:2.0:1767319489.712109:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920563584:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319489.712134:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319489.712237:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920563584:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 128us (333us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319489.712246:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5135 00000100:00100000:1.0:1767319489.714105:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319489.714228:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920563712 00000100:00100000:3.0:1767319489.714250:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920563712 00000100:00100000:3.0:1767319489.714362:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4649 00000100:00100000:3.0:1767319489.714374:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920563712:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319489.714611:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920563712:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 239us (525us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319489.714621:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4649 00000100:00080000:2.0:1767319489.714714:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319490.398858:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319490.779420:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319490.779431:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319490.779434:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319490.779438:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319490.779442:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319490.779450:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4733 00000100:00080000:3.0:1767319490.779454:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319490.779460:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319490.779499:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7480 x1853163985182976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319490.779534:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319490.779538:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319490.779539:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319490.779542:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4733 00000100:00080000:3.0:1767319490.779544:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319490.779547:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319490.779558:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985183104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319490.779593:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319490.779597:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319490.779599:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319490.779602:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4733 00000100:00080000:3.0:1767319490.779605:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319490.779609:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319490.779622:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7100 x1853163985183360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319490.779649:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319490.779652:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319490.779654:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319490.779663:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4733 00000100:00100000:1.0:1767319490.779669:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985183232:0@lo:400:kworker.0 00000100:00080000:3.0:1767319490.779669:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319490.779678:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319490.779699:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6a00 x1853163985183616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319490.779776:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985183872:0@lo:400:kworker.0 00000100:00100000:1.0:1767319490.779801:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319490.779937:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985183488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319490.779972:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319465/real 1767319465] req@ffff8ce1b5ba4a80 x1853163985176448/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319490 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319490.780001:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5ba4a80 x1853163985176448/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319490 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319490.780021:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985182976:0@lo:38:kworker.0 00000100:00100000:3.0:1767319490.780792:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319490.780871:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985183872 00000100:00100000:3.0:1767319490.780879:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985183872 00000100:00100000:3.0:1767319490.780927:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3059 00000100:00100000:3.0:1767319490.780936:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985183872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319490.780993:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985183872:12345-0@lo:400:kworker.0 Request processed in 58us (203us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319490.781001:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3059 00000100:00100000:3.0:1767319490.781040:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985183872:0@lo:400:kworker.0 00000100:00100000:3.0:1767319490.781089:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985183744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319490.781129:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319490.781155:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985183744 00000100:00100000:3.0:1767319490.781704:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985183744 00000100:00100000:3.0:1767319490.781754:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3060 00000100:00100000:3.0:1767319490.781765:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985183744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319490.781771:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985183232 00000100:00100000:1.0:1767319490.781784:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985183232 00000100:00100000:3.0:1767319490.781814:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985183744:12345-0@lo:400:kworker.0 Request processed in 51us (695us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319490.781821:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3060 00000100:00100000:1.0:1767319490.781874:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4650 00000100:00100000:3.0:1767319490.781882:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985183744:0@lo:400:kworker.0 00000100:00080000:3.0:1767319490.781917:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319490.781951:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985183232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319490.782050:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985183232:12345-0@lo:400:kworker.0 Request processed in 166us (2250us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319490.782058:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4650 00000100:00100000:3.0:1767319490.782209:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985183232:0@lo:400:kworker.0 00000100:00100000:0.0:1767319490.784659:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319490.784885:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985183488 00000100:00100000:1.0:1767319490.784907:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985183488 00000100:00100000:1.0:1767319490.785014:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4651 00000100:00100000:1.0:1767319490.785027:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985183488:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319490.785387:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319490.785522:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985183488:12345-0@lo:400:kworker.0 Request processed in 496us (872us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319490.785535:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4651 00000100:00100000:0.0:1767319490.785633:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985183488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319490.792216:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319490.792289:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985183104:0@lo:38:kworker.0 00000100:00100000:2.0:1767319490.792344:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319490.792371:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985183360:0@lo:38:kworker.0 00000100:00100000:2.0:1767319490.792394:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319490.792460:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985182976 00000100:00100000:1.0:1767319490.792476:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985182976 00000100:00100000:1.0:1767319490.792531:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985183360 00000100:00100000:1.0:1767319490.792534:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985183360 00000100:00100000:1.0:1767319490.792554:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5136 00000100:00100000:1.0:1767319490.792563:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985182976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319490.792591:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319490.792669:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985182976:12345-0@lo:38:kworker.0 Request processed in 106us (464us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319490.792678:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5136 00000100:00100000:1.0:1767319490.792695:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5137 00000100:00100000:1.0:1767319490.792699:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985183360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319490.792709:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319490.792744:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985183360:12345-0@lo:38:kworker.0 Request processed in 43us (349us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319490.792750:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5137 00000100:00100000:0.0:1767319490.792997:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985183104 00000100:00100000:0.0:1767319490.793004:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985183104 00000100:00100000:0.0:1767319490.793042:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1774 00000100:00100000:0.0:1767319490.793047:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985183104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319490.793057:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319490.793095:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985183104:12345-0@lo:38:kworker.0 Request processed in 46us (751us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319490.793103:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1774 00000100:00100000:2.0:1767319490.794217:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985183616:0@lo:38:kworker.0 00000100:00100000:2.0:1767319490.794291:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319490.794329:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319490.794348:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319490.794352:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985176448:0@lo:250:kworker.0 00000100:00080000:2.0:1767319490.794394:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319490.794398:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319490.794401:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985182976:0@lo:38:kworker.0 00000100:00080000:2.0:1767319490.794410:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319490.794413:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319490.794416:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985183104:0@lo:38:kworker.0 00000100:00080000:2.0:1767319490.794439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319490.794443:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319490.794446:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985183360:0@lo:38:kworker.0 00000100:00080000:3.0:1767319490.794757:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319490.794767:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319490.794772:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319490.794781:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4713 00000100:00080000:3.0:1767319490.794786:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319490.794794:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319490.794836:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6d80 x1853163985184000/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319490.794972:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985184000:0@lo:250:kworker.0 00000400:00000100:2.0:1767319490.794997:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985184000 offset 0 length 520: 4 00000100:00100000:0.0:1767319490.795548:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985183616 00000100:00100000:0.0:1767319490.795560:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985183616 00000100:00100000:0.0:1767319490.795618:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5138 00000100:00100000:0.0:1767319490.795627:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985183616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319490.795651:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319490.795750:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985183616:12345-0@lo:38:kworker.0 Request processed in 123us (1457us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319490.795759:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5138 00000100:00080000:2.0:1767319490.796074:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319490.796081:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319490.796086:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985183616:0@lo:38:kworker.0 00010000:00010000:1.0:1767319491.419882:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319492.443757:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319493.471282:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319494.367504:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319494.367540:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985184128:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319494.367632:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319494.367708:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319494.367716:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985184128 00000100:00100000:3.0:1767319494.367727:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985184128 00000100:00100000:0.0:1767319494.367733:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985184256:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319494.367800:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2226 00000100:00100000:0.0:1767319494.367807:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319494.367818:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985184128:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319494.367853:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319494.367887:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985184256 00000100:00100000:0.0:1767319494.367896:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985184256 00000100:00100000:3.0:1767319494.367919:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985184128:12345-0@lo:13:osp-pre-1-1.0 Request processed in 106us (286us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319494.367926:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2226 00000100:00100000:3.0:1767319494.367951:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2227 00000100:00100000:3.0:1767319494.367962:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985184256:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319494.367968:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319494.367976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:3.0:1767319494.367983:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319494.368040:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985184256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319494.368047:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2227 00000100:00100000:3.0:1767319494.368094:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985184128:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319494.368335:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985184256:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319494.494195:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319494.827099:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319494.828066:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920563968 00000100:00100000:3.0:1767319494.828078:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920563968 00000100:00100000:3.0:1767319494.828205:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4652 00000100:00100000:3.0:1767319494.828215:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920563968:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319494.828313:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920563968:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (1226us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319494.828320:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4652 00000100:00080000:0.0:1767319494.828760:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319494.831304:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319494.832121:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920563840 00000100:00100000:0.0:1767319494.832133:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920563840 00000100:00100000:0.0:1767319494.832207:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5139 00000100:00100000:0.0:1767319494.832215:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920563840:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319494.832239:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319494.832803:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920563840:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 585us (1509us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319494.832816:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5139 00010000:00010000:0.0:1767319495.515867:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319495.900228:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319495.900240:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319495.900244:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319495.900249:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319495.900253:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319495.900262:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4739 00000100:00080000:0.0:1767319495.900267:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319495.900273:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319495.900314:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7100 x1853163985184384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319495.900340:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319495.900343:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319495.900345:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319495.900348:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4739 00000100:00080000:0.0:1767319495.900350:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319495.900353:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319495.900367:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5f80 x1853163985184512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319495.900396:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319495.900400:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319495.900401:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319495.900405:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4739 00000100:00080000:0.0:1767319495.900408:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319495.900411:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319495.900421:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7b80 x1853163985184768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319495.900446:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319495.900450:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319495.900451:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319495.900457:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4739 00000100:00100000:2.0:1767319495.900457:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985184384:0@lo:38:kworker.0 00000100:00080000:0.0:1767319495.900460:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319495.900467:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319495.900481:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631500 x1853163985185024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319495.900538:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985184640:0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.900546:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319495.900583:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985185152:0@lo:400:kworker.0 00000100:00100000:0.0:1767319495.900608:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319495.900685:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319495.900691:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985184896:0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.903690:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985184512:0@lo:38:kworker.0 00000100:00100000:1.0:1767319495.903697:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319495.903735:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319495.903744:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985185280:0@lo:400:kworker.0 00000100:00100000:3.0:1767319495.903851:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185152 00000100:00100000:3.0:1767319495.903864:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185152 00000100:00100000:3.0:1767319495.903915:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3061 00000100:00100000:3.0:1767319495.903925:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985185152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.903991:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319495.904025:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319495.904025:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985184768:0@lo:38:kworker.0 00000100:00100000:2.0:1767319495.904063:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319495.904069:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185280 00000100:00100000:1.0:1767319495.904073:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185280 00000100:00100000:2.0:1767319495.904095:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985185024:0@lo:38:kworker.0 00000100:00100000:1.0:1767319495.904105:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3062 00000100:00100000:1.0:1767319495.904114:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985185280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.904127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319495.904192:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985184384 00000100:00100000:2.0:1767319495.904197:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985184384 00000100:00100000:1.0:1767319495.904201:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985185280:12345-0@lo:400:kworker.0 Request processed in 87us (187us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319495.904209:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3062 00000100:00100000:2.0:1767319495.904223:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985184640 00000100:00100000:2.0:1767319495.904228:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985184640 00000100:00100000:1.0:1767319495.904237:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985184768 00000100:00100000:1.0:1767319495.904242:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985184768 00000100:00100000:2.0:1767319495.904266:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185024 00000100:00100000:2.0:1767319495.904271:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185024 00000100:00100000:1.0:1767319495.904275:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4653 00000100:00100000:1.0:1767319495.904285:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985184640:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.904291:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5140 00000100:00100000:2.0:1767319495.904297:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985184384:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319495.904318:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985184640:12345-0@lo:400:kworker.0 Request processed in 35us (624us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319495.904327:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4653 00010000:00020000:2.0:1767319495.904327:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319495.904344:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5141 00000100:00100000:1.0:1767319495.904350:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985184768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319495.904361:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319495.904373:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985184384:12345-0@lo:38:kworker.0 Request processed in 72us (3830us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319495.904385:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5140 00000100:00100000:1.0:1767319495.904392:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985184768:12345-0@lo:38:kworker.0 Request processed in 40us (329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319495.904400:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5141 00000100:00100000:2.0:1767319495.904410:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5142 00000100:00100000:2.0:1767319495.904416:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985185024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319495.904427:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319495.904452:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985185280:0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.904456:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985185024:12345-0@lo:38:kworker.0 Request processed in 38us (328us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319495.904462:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5142 00000100:00080000:2.0:1767319495.904479:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319495.904497:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319495.904502:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319495.904505:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985185024:0@lo:38:kworker.0 00000100:00080000:2.0:1767319495.904513:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319495.904517:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319495.904520:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985184384:0@lo:38:kworker.0 00000100:00080000:2.0:1767319495.904526:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319495.904529:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319495.904531:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985184768:0@lo:38:kworker.0 00000100:00100000:2.0:1767319495.904594:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985184512 00000100:00100000:2.0:1767319495.904597:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985184512 00000100:00100000:2.0:1767319495.904618:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1775 00000100:00100000:2.0:1767319495.904623:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190a43050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985184512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319495.904646:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319495.904675:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190a43050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985184512:12345-0@lo:38:kworker.0 Request processed in 51us (685us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319495.904684:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1775 00000100:00080000:2.0:1767319495.904845:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319495.904849:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319495.904852:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985184512:0@lo:38:kworker.0 00000100:00100000:1.0:1767319495.905417:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985184640:0@lo:400:kworker.0 00000100:00100000:0.0:1767319495.909337:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319495.909622:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985184896 00000100:00100000:2.0:1767319495.909644:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985184896 00000100:00100000:2.0:1767319495.909729:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4654 00000100:00100000:2.0:1767319495.909739:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985184896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.909814:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985184896:12345-0@lo:400:kworker.0 Request processed in 77us (486us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319495.909821:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4654 00000100:00080000:2.0:1767319495.909842:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319495.910020:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985184896:0@lo:400:kworker.0 00000100:00100000:2.0:1767319495.912497:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985185152:12345-0@lo:400:kworker.0 Request processed in 8561us (11874us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319495.912513:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3061 00000100:00100000:0.0:1767319495.912600:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985185152:0@lo:400:kworker.0 00010000:00010000:1.0:1767319496.541053:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319497.563359:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319498.587447:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319499.483850:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319499.483873:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985185408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319499.483948:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319499.484041:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319499.484054:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985185536:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319499.484084:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319499.484186:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185408 00000100:00100000:2.0:1767319499.484195:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185408 00000100:00100000:2.0:1767319499.484256:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185536 00000100:00100000:2.0:1767319499.484260:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185536 00000100:00100000:2.0:1767319499.484284:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2228 00000100:00100000:2.0:1767319499.484293:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985185408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319499.484323:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319499.484364:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985185408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (415us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319499.484372:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2228 00000100:00100000:2.0:1767319499.484390:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2229 00000100:00100000:2.0:1767319499.484396:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985185536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319499.484410:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319499.484439:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985185536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (353us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319499.484445:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2229 00000100:00100000:2.0:1767319499.484502:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985185408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319499.484532:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985185536:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319499.484855:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319499.484860:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319499.611624:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319499.948865:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319499.948929:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319499.949199:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920564224 00000100:00100000:3.0:1767319499.949209:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920564224 00000100:00100000:3.0:1767319499.949282:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920564096 00000100:00100000:3.0:1767319499.949286:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920564096 00000100:00100000:3.0:1767319499.949306:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4655 00000100:00100000:3.0:1767319499.949317:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920564224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319499.949389:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920564224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (534us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319499.949397:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4655 00000100:00100000:3.0:1767319499.949415:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5143 00000100:00100000:3.0:1767319499.949420:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920564096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319499.949440:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319499.949465:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920564096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (537us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319499.949475:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5143 00000100:00080000:2.0:1767319499.951346:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319500.636876:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319501.024636:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319501.024647:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319501.024650:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319501.024662:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319501.024667:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319501.024676:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4744 00000100:00080000:2.0:1767319501.024680:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319501.024695:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319501.024736:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73d500 x1853163985185664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319501.024766:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319501.024771:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319501.024772:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319501.024775:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4744 00000100:00080000:2.0:1767319501.024777:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319501.024781:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319501.024794:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffd880 x1853163985185792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319501.024839:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319501.024843:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319501.024845:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319501.024848:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4744 00000100:00080000:2.0:1767319501.024850:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319501.024860:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319501.024871:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13100 x1853163985186048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319501.024897:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319501.024900:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319501.024902:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319501.024905:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4744 00000100:00080000:2.0:1767319501.024907:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319501.024910:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319501.024919:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b480 x1853163985186304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319501.024973:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985186176:0@lo:400:kworker.0 00000100:00100000:2.0:1767319501.025035:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319501.025105:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319501.025111:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985186432:0@lo:400:kworker.0 00000100:00100000:2.0:1767319501.025139:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319501.025153:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985185920:0@lo:400:kworker.0 00000100:00100000:2.0:1767319501.025239:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186432 00000100:00100000:2.0:1767319501.025249:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186432 00000100:00100000:3.0:1767319501.025302:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319501.025318:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3063 00000100:00100000:2.0:1767319501.025334:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985186432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319501.025371:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319501.025383:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985186560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319501.025401:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985186432:12345-0@lo:400:kworker.0 Request processed in 71us (259us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319501.025411:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3063 00000100:00100000:3.0:1767319501.025429:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319501.025476:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186176 00000100:00100000:3.0:1767319501.025484:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186176 00000100:00100000:0.0:1767319501.025508:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985185664:0@lo:38:kworker.0 00000100:00100000:3.0:1767319501.025548:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185920 00000100:00100000:3.0:1767319501.025554:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185920 00000100:00100000:0.0:1767319501.025587:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319501.025594:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4656 00000100:00100000:3.0:1767319501.025607:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985186176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319501.025628:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985185792:0@lo:38:kworker.0 00000100:00100000:1.0:1767319501.025737:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185664 00000100:00100000:1.0:1767319501.025743:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185664 00000100:00100000:1.0:1767319501.025770:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4657 00000100:00100000:1.0:1767319501.025779:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985185920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319501.026123:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186560 00000100:00100000:2.0:1767319501.026128:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186560 00000100:00100000:2.0:1767319501.026156:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3064 00000100:00100000:2.0:1767319501.026161:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985186560:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319501.031652:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985186176:12345-0@lo:400:kworker.0 Request processed in 6031us (6598us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319501.031673:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4656 00000100:00100000:3.0:1767319501.031733:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5144 00000100:00100000:3.0:1767319501.031742:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985185664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319501.031772:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319501.034236:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985185664:12345-0@lo:38:kworker.0 Request processed in 2079us (8238us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319501.034254:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5144 00000100:00080000:3.0:1767319501.034500:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319501.034507:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319501.034511:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319501.034586:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985186432:0@lo:400:kworker.0 00000100:00100000:3.0:1767319501.034607:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985186176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319501.034728:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319501.034823:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985186048:0@lo:38:kworker.0 00000100:00100000:0.0:1767319501.034896:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319501.034917:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985186304:0@lo:38:kworker.0 00000100:00100000:0.0:1767319501.034948:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319501.034998:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319501.035004:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319501.035008:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985185664:0@lo:38:kworker.0 00000100:00100000:0.0:1767319501.035049:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186048 00000100:00100000:0.0:1767319501.035055:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186048 00000100:00100000:0.0:1767319501.035084:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186304 00000100:00100000:0.0:1767319501.035086:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186304 00000100:00100000:0.0:1767319501.035102:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5145 00000100:00100000:0.0:1767319501.035108:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985186048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319501.035128:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319501.035190:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985186048:12345-0@lo:38:kworker.0 Request processed in 80us (292us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319501.035197:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5145 00000100:00100000:0.0:1767319501.035212:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5146 00000100:00100000:0.0:1767319501.035216:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985186304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319501.035224:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319501.035247:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985186304:12345-0@lo:38:kworker.0 Request processed in 30us (309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319501.035252:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5146 00000100:00080000:0.0:1767319501.035285:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319501.035289:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319501.035291:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985186048:0@lo:38:kworker.0 00000100:00080000:0.0:1767319501.035298:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319501.035300:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319501.035302:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985186304:0@lo:38:kworker.0 00000100:00100000:3.0:1767319501.037026:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985185792 00000100:00100000:3.0:1767319501.037043:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985185792 00000100:00100000:3.0:1767319501.037121:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1776 00000100:00100000:3.0:1767319501.037130:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985185792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319501.037155:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319501.037286:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985185792:12345-0@lo:38:kworker.0 Request processed in 156us (2569us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319501.037294:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1776 00000100:00080000:0.0:1767319501.038207:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319501.038220:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319501.038226:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985185792:0@lo:38:kworker.0 00000100:00100000:1.0:1767319501.040934:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985185920:12345-0@lo:400:kworker.0 Request processed in 15147us (15624us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319501.040957:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4657 00000100:00100000:3.0:1767319501.043409:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985185920:0@lo:400:kworker.0 00000100:00100000:2.0:1767319501.043788:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985186560:12345-0@lo:400:kworker.0 Request processed in 17616us (18354us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319501.043803:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3064 00000100:00100000:3.0:1767319501.045568:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985186560:0@lo:400:kworker.0 00010000:00010000:0.0:1767319501.659776:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319502.683461:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319503.708266:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319504.604039:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985186688:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319504.604117:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319504.605657:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985186816:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319504.605742:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319504.607233:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186688 00000100:00100000:0.0:1767319504.607248:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186688 00000100:00100000:0.0:1767319504.607339:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186816 00000100:00100000:0.0:1767319504.607343:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186816 00000100:00100000:0.0:1767319504.607374:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2230 00000100:00100000:0.0:1767319504.607385:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985186688:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319504.607506:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319504.607512:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319504.607554:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2231 00000100:00100000:3.0:1767319504.607565:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985186816:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319504.607604:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319504.607692:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985186816:12345-0@lo:13:osp-pre-0-1.0 Request processed in 130us (1949us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319504.607700:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2231 00000100:00100000:3.0:1767319504.607767:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985186816:0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319504.607842:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319504.617856:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985186688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 10458us (13722us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319504.617879:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2230 00000100:00100000:2.0:1767319504.617960:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985186688:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319504.733259:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319505.069641:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319505.069684:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319505.069781:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920564480 00000100:00100000:0.0:1767319505.069790:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920564480 00000100:00100000:0.0:1767319505.069855:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920564352 00000100:00100000:0.0:1767319505.069859:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920564352 00000100:00100000:0.0:1767319505.069880:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4658 00000100:00100000:0.0:1767319505.069890:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920564480:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319505.069961:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920564480:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 72us (330us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319505.069968:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4658 00000100:00100000:0.0:1767319505.069988:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5147 00000100:00100000:0.0:1767319505.069993:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920564352:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319505.070013:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319505.070038:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920564352:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (353us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319505.070044:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5147 00000100:00080000:3.0:1767319505.070195:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319505.760710:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319506.140834:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319506.140849:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319506.140853:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319506.140858:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319506.140864:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319506.140875:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4749 00000100:00080000:0.0:1767319506.140881:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319506.140891:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319506.141067:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1880 x1853163985186944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319506.141089:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319506.141092:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319506.141095:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319506.141099:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4749 00000100:00080000:0.0:1767319506.141101:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319506.141105:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319506.141121:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0e00 x1853163985187072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319506.141531:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319506.141600:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985186944:0@lo:38:kworker.0 00000100:00100000:2.0:1767319506.141690:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319506.141761:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985187072:0@lo:38:kworker.0 00000100:00100000:2.0:1767319506.141793:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319506.141844:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187072 00000100:00100000:2.0:1767319506.141853:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187072 00000100:00100000:2.0:1767319506.141895:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1777 00000100:00100000:2.0:1767319506.141919:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985187072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319506.141945:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319506.141993:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985187072:12345-0@lo:38:kworker.0 Request processed in 75us (198us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319506.142001:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1777 00000100:00080000:2.0:1767319506.142049:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319506.142054:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319506.142058:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985187072:0@lo:38:kworker.0 00000100:00100000:2.0:1767319506.142104:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985186944 00000100:00100000:2.0:1767319506.142108:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985186944 00000100:00100000:2.0:1767319506.142131:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5148 00000100:00100000:2.0:1767319506.142147:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985186944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319506.142157:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319506.142212:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985186944:12345-0@lo:38:kworker.0 Request processed in 75us (522us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319506.142218:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5148 00000100:00080000:2.0:1767319506.142240:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319506.142244:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319506.142247:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985186944:0@lo:38:kworker.0 00000100:00080000:0.0:1767319506.142995:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319506.143001:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319506.143004:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319506.143010:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4749 00000100:00080000:0.0:1767319506.143014:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319506.143019:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319506.143047:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2680 x1853163985187328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319506.143090:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319506.143093:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319506.143095:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319506.143099:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4749 00000100:00080000:0.0:1767319506.143101:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319506.143104:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319506.143115:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0e00 x1853163985187584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319506.143193:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985187328:0@lo:38:kworker.0 00000100:00100000:2.0:1767319506.143238:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319506.143265:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985187584:0@lo:38:kworker.0 00000100:00100000:2.0:1767319506.143285:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319506.143331:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187328 00000100:00100000:2.0:1767319506.143337:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187328 00000100:00100000:2.0:1767319506.143359:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187584 00000100:00100000:2.0:1767319506.143362:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187584 00000100:00100000:2.0:1767319506.143391:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5149 00000100:00100000:2.0:1767319506.143397:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985187328:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319506.143411:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319506.143456:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985187328:12345-0@lo:38:kworker.0 Request processed in 57us (216us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319506.143462:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5149 00000100:00080000:2.0:1767319506.143488:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319506.143492:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319506.143495:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985187328:0@lo:38:kworker.0 00000100:00100000:2.0:1767319506.143516:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5150 00000100:00100000:2.0:1767319506.143798:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985187584:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319506.143809:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319506.143843:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985187584:12345-0@lo:38:kworker.0 Request processed in 47us (556us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319506.143849:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5150 00000100:00080000:2.0:1767319506.143870:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319506.143874:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319506.143877:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985187584:0@lo:38:kworker.0 00000100:00100000:2.0:1767319506.143932:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985187712:0@lo:400:kworker.0 00000100:00100000:2.0:1767319506.143965:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319506.143988:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319506.143992:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985187456:0@lo:400:kworker.0 00000100:00100000:2.0:1767319506.144011:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319506.144039:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187712 00000100:00100000:2.0:1767319506.144043:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187712 00000100:00100000:2.0:1767319506.144100:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3065 00000100:00100000:2.0:1767319506.144109:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985187712:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319506.144189:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985187712:12345-0@lo:400:kworker.0 Request processed in 56us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319506.144197:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3065 00000100:00100000:2.0:1767319506.144227:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985187712:0@lo:400:kworker.0 00000100:00100000:2.0:1767319506.144243:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187456 00000100:00100000:2.0:1767319506.144246:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187456 00000100:00100000:2.0:1767319506.144273:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4659 00000100:00100000:2.0:1767319506.144279:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985187456:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319506.144403:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319506.144414:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985187840:0@lo:400:kworker.0 00000100:00100000:1.0:1767319506.144455:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319506.144487:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187840 00000100:00100000:1.0:1767319506.144489:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187840 00000100:00100000:1.0:1767319506.144514:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3066 00000100:00100000:1.0:1767319506.144533:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985187840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319506.144562:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985187840:12345-0@lo:400:kworker.0 Request processed in 43us (107us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319506.144568:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3066 00000100:00100000:1.0:1767319506.144591:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985187840:0@lo:400:kworker.0 00000100:00080000:1.0:1767319506.144649:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319506.144654:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319506.144655:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319506.145252:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985187456:12345-0@lo:400:kworker.0 Request processed in 970us (1236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319506.145260:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4659 00000100:00100000:1.0:1767319506.145581:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985187456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319506.145802:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985187200:0@lo:400:kworker.0 00000100:00100000:3.0:1767319506.145847:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319506.146033:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187200 00000100:00100000:2.0:1767319506.146039:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187200 00000100:00100000:2.0:1767319506.146186:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4660 00000100:00100000:2.0:1767319506.146194:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985187200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319506.146238:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985187200:12345-0@lo:400:kworker.0 Request processed in 45us (391us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319506.146245:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4660 00000100:00100000:3.0:1767319506.146733:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985187200:0@lo:400:kworker.0 00000100:00080000:1.0:1767319506.146796:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319506.782544:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319507.804583:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319508.827484:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319509.723418:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985187968:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319509.723432:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319509.723448:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985188096:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319509.723490:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319509.723621:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985187968 00000100:00100000:0.0:1767319509.723630:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985187968 00000100:00080000:2.0:1767319509.723724:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319509.723868:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319509.730407:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188096 00000100:00100000:0.0:1767319509.730421:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188096 00000100:00100000:0.0:1767319509.730513:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2232 00000100:00100000:0.0:1767319509.730525:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985187968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319509.730559:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319509.730654:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985187968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 119us (7152us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319509.730662:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2232 00000100:00100000:0.0:1767319509.730682:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2233 00000100:00100000:0.0:1767319509.730689:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985188096:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319509.730704:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319509.730737:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985188096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (6868us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319509.730743:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2233 00000100:00100000:0.0:1767319509.730797:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985187968:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319509.730873:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985188096:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319509.732331:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319509.851444:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319510.186284:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319510.186337:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319510.186435:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920564608 00000100:00100000:2.0:1767319510.186446:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920564608 00000100:00100000:2.0:1767319510.186491:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920564736 00000100:00100000:2.0:1767319510.186495:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920564736 00000100:00100000:2.0:1767319510.186546:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4661 00000100:00100000:2.0:1767319510.186556:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920564736:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319510.186630:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920564736:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319510.186638:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4661 00000100:00100000:2.0:1767319510.186672:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5151 00000100:00100000:2.0:1767319510.186677:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920564608:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319510.186697:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319510.186722:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920564608:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (453us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319510.186727:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5151 00000100:00080000:0.0:1767319510.187467:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319510.876661:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319511.259463:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319511.259473:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319511.259477:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319511.259481:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319511.259486:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319511.259494:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4754 00000100:00080000:1.0:1767319511.259499:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319511.259506:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319511.259758:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985188224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319511.259783:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319511.259787:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319511.259789:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319511.259793:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4754 00000100:00080000:1.0:1767319511.259795:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319511.259800:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319511.259812:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985188352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319511.259886:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985188480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319511.259952:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319511.260047:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985188224:0@lo:38:kworker.0 00000100:00100000:2.0:1767319511.260078:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319511.260101:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985188352:0@lo:38:kworker.0 00000100:00100000:2.0:1767319511.260122:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319511.260157:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188480 00000100:00100000:2.0:1767319511.260177:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188480 00000100:00100000:2.0:1767319511.260224:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188224 00000100:00100000:2.0:1767319511.260228:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188224 00000100:00100000:2.0:1767319511.260242:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4662 00000100:00100000:2.0:1767319511.260251:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985188480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.260335:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5152 00000100:00100000:3.0:1767319511.260341:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985188224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319511.260369:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319511.260406:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985188224:12345-0@lo:38:kworker.0 Request processed in 63us (326us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319511.260413:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5152 00000100:00100000:3.0:1767319511.260457:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188352 00000100:00100000:3.0:1767319511.260460:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188352 00000100:00100000:3.0:1767319511.260478:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1778 00000100:00100000:3.0:1767319511.260482:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985188352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319511.260492:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319511.260527:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985188352:12345-0@lo:38:kworker.0 Request processed in 30us (391us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319511.260533:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1778 00000100:00080000:3.0:1767319511.260572:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319511.260577:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319511.260581:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985188224:0@lo:38:kworker.0 00000100:00080000:3.0:1767319511.260590:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319511.260593:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319511.260595:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985188352:0@lo:38:kworker.0 00000100:00080000:3.0:1767319511.260642:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319511.262492:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319511.262500:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319511.262504:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319511.262521:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4754 00000100:00080000:1.0:1767319511.262525:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319511.262531:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319511.262563:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05c00 x1853163985188608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319511.262601:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319511.262605:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319511.262607:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319511.262623:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4754 00000100:00080000:1.0:1767319511.262626:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319511.262629:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319511.262639:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04700 x1853163985188864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319511.262722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985188608:0@lo:38:kworker.0 00000100:00100000:3.0:1767319511.262775:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319511.262803:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985188864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319511.262840:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319511.262879:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188608 00000100:00100000:3.0:1767319511.262886:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188608 00000100:00100000:3.0:1767319511.262913:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188864 00000100:00100000:3.0:1767319511.262916:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188864 00000100:00100000:3.0:1767319511.262934:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5153 00000100:00100000:3.0:1767319511.262941:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985188608:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319511.262959:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319511.262994:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985188608:12345-0@lo:38:kworker.0 Request processed in 53us (219us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319511.263001:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5153 00000100:00100000:3.0:1767319511.263026:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5154 00000100:00100000:3.0:1767319511.263031:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985188864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319511.263041:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319511.263063:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985188864:12345-0@lo:38:kworker.0 Request processed in 31us (223us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319511.263069:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5154 00000100:00080000:3.0:1767319511.263104:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319511.263109:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319511.263113:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985188608:0@lo:38:kworker.0 00000100:00080000:3.0:1767319511.263120:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319511.263123:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319511.263125:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985188864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319511.263147:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985189120:0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263196:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319511.263241:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985188736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263263:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319511.263280:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319511.263285:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985188992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263304:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319511.263324:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188736 00000100:00100000:3.0:1767319511.263327:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188736 00000100:00100000:3.0:1767319511.263374:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4663 00000100:00100000:3.0:1767319511.263380:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985188736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263417:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985188736:12345-0@lo:400:kworker.0 Request processed in 37us (153us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319511.263424:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4663 00000100:00080000:3.0:1767319511.263439:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319511.263460:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985188736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263486:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985189120 00000100:00100000:3.0:1767319511.263489:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985189120 00000100:00100000:3.0:1767319511.263515:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985188992 00000100:00100000:3.0:1767319511.263518:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985188992 00000100:00100000:3.0:1767319511.263533:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3067 00000100:00100000:3.0:1767319511.263539:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985189120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263569:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985189120:12345-0@lo:400:kworker.0 Request processed in 30us (372us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319511.263575:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3067 00000100:00100000:3.0:1767319511.263586:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3068 00000100:00100000:3.0:1767319511.263592:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985188992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263618:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985188992:12345-0@lo:400:kworker.0 Request processed in 26us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319511.263624:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3068 00000100:00080000:3.0:1767319511.263642:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319511.263645:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319511.263659:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985189120:0@lo:400:kworker.0 00000100:00100000:3.0:1767319511.263676:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985188992:0@lo:400:kworker.0 00000100:00100000:2.0:1767319511.270741:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985188480:12345-0@lo:400:kworker.0 Request processed in 10481us (10778us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319511.270756:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4662 00000100:00100000:0.0:1767319511.277281:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985188480:0@lo:400:kworker.0 00010000:00010000:1.0:1767319511.899590:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319512.927991:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319513.947483:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319514.285328:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319514.285367:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920564864 offset 0 length 520: 4 00000100:00100000:2.0:1767319514.285393:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319514.285472:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920565120 00000100:00100000:3.0:1767319514.285481:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920565120 00000100:00100000:3.0:1767319514.285549:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920564992 00000100:00100000:3.0:1767319514.285553:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920564992 00000100:00100000:3.0:1767319514.285574:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4664 00000100:00100000:3.0:1767319514.285583:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920565120:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319514.285653:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920565120:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319514.285660:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4664 00000100:00100000:3.0:1767319514.285677:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5155 00000100:00100000:3.0:1767319514.285681:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920564992:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319514.285701:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319514.285725:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920564992:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (331us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319514.285730:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5155 00000100:00080000:0.0:1767319514.288392:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319514.844480:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319514.844503:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985189376:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319514.844567:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319514.845002:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985189376 00000100:00100000:2.0:1767319514.845010:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985189376 00000100:00100000:2.0:1767319514.845077:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2234 00000100:00100000:2.0:1767319514.845087:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985189376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319514.845115:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319514.845203:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985189376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (595us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319514.845212:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2234 00000100:00100000:2.0:1767319514.845234:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319514.845243:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985189248:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319514.845271:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319514.845310:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985189248 00000100:00100000:2.0:1767319514.845314:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985189248 00000100:00100000:2.0:1767319514.845336:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2235 00000100:00100000:2.0:1767319514.845343:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985189248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319514.845359:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319514.845387:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985189248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 45us (115us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319514.845397:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2235 00000100:00100000:2.0:1767319514.845433:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985189248:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319514.845583:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985189376:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319514.845770:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319514.845774:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319514.971475:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319515.355361:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319490/real 1767319490] req@ffff8ce19dca6d80 x1853163985184000/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319515 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319515.355406:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19dca6d80 x1853163985184000/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319515 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319515.355425:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319515.355448:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319515.355452:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985184000:0@lo:250:kworker.0 00000100:00080000:1.0:1767319515.355525:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319515.355530:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319515.355533:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319515.355537:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4739 00000100:00080000:1.0:1767319515.355542:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319515.355548:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319515.355584:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05880 x1853163985189504/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319515.355609:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319515.355613:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319515.355615:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319515.355618:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4759 00000100:00080000:1.0:1767319515.355621:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319515.355624:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319515.355639:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985189632/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319515.355647:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319515.355650:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319515.355652:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319515.355655:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4759 00000100:00080000:1.0:1767319515.355658:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319515.355661:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319515.355672:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07480 x1853163985189760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319515.355717:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319515.355721:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319515.355723:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319515.355726:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4759 00000100:00080000:1.0:1767319515.355728:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319515.355732:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319515.355742:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04700 x1853163985190016/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319515.355746:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985189888:0@lo:400:kworker.0 00000100:00080000:1.0:1767319515.355766:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319515.355770:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319515.355771:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319515.355774:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4759 00000100:00080000:1.0:1767319515.355776:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319515.355779:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319515.355787:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bebb80 x1853163985190272/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319515.355804:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319515.355806:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319515.355815:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985190144:0@lo:400:kworker.0 00000100:00100000:1.0:1767319515.355839:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319515.355870:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985189888 00000100:00100000:3.0:1767319515.355877:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985189888 00000100:00100000:3.0:1767319515.355934:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190144 00000100:00100000:3.0:1767319515.355937:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190144 00000100:00100000:3.0:1767319515.355970:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4665 00000100:00100000:3.0:1767319515.355979:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985189888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319515.356027:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985189888:12345-0@lo:400:kworker.0 Request processed in 49us (222us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319515.356035:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4665 00000100:00100000:2.0:1767319515.356046:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4666 00000100:00100000:2.0:1767319515.356055:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985190144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319515.356082:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985189888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319515.356091:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985190144:12345-0@lo:400:kworker.0 Request processed in 38us (251us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319515.356097:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4666 00000100:00100000:2.0:1767319515.356133:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985190144:0@lo:400:kworker.0 00000100:00080000:1.0:1767319515.356317:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319515.356319:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985189504:0@lo:250:kworker.0 00000400:00000100:0.0:1767319515.356343:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985189504 offset 0 length 520: 4 00000100:00100000:0.0:1767319515.356362:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985189632:0@lo:38:kworker.0 00000100:00100000:0.0:1767319515.356382:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319515.356413:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985189760:0@lo:38:kworker.0 00000100:00100000:0.0:1767319515.356437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319515.356460:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985190016:0@lo:38:kworker.0 00000100:00100000:0.0:1767319515.356479:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319515.356488:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985189760 00000100:00100000:3.0:1767319515.356492:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985189760 00000100:00100000:0.0:1767319515.356502:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985190272:0@lo:38:kworker.0 00000100:00100000:3.0:1767319515.356515:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1779 00000100:00100000:0.0:1767319515.356520:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319515.356520:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985189760:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319515.356538:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319515.356545:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985189632 00000100:00100000:0.0:1767319515.356549:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985189632 00000100:00100000:0.0:1767319515.356567:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190016 00000100:00100000:0.0:1767319515.356570:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190016 00000100:00100000:0.0:1767319515.356582:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190272 00000100:00100000:0.0:1767319515.356584:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190272 00000100:00100000:3.0:1767319515.356584:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985189760:12345-0@lo:38:kworker.0 Request processed in 62us (146us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319515.356590:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1779 00000100:00100000:0.0:1767319515.356595:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5156 00000100:00100000:0.0:1767319515.356599:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985189632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319515.356611:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319515.356619:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5157 00000100:00100000:3.0:1767319515.356623:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985190016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319515.356632:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319515.356635:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985189632:12345-0@lo:38:kworker.0 Request processed in 35us (253us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319515.356641:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5156 00000100:00100000:0.0:1767319515.356653:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5158 00000100:00100000:3.0:1767319515.356653:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985190016:12345-0@lo:38:kworker.0 Request processed in 28us (172us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319515.356657:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985190272:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319515.356658:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5157 00010000:00020000:0.0:1767319515.356665:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319515.356686:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985190272:12345-0@lo:38:kworker.0 Request processed in 27us (164us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319515.356692:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5158 00000100:00080000:0.0:1767319515.356715:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319515.356719:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319515.356722:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985189632:0@lo:38:kworker.0 00000100:00080000:0.0:1767319515.356729:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319515.356732:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319515.356734:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985189760:0@lo:38:kworker.0 00000100:00080000:0.0:1767319515.356741:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319515.356744:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319515.356746:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985190016:0@lo:38:kworker.0 00000100:00080000:0.0:1767319515.356753:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319515.356756:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319515.356758:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985190272:0@lo:38:kworker.0 00010000:00010000:1.0:1767319515.995748:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319517.019575:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319518.043684:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319519.067565:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319519.403881:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319519.403934:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319519.404026:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920565248 00000100:00100000:0.0:1767319519.404037:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920565248 00000100:00100000:0.0:1767319519.404081:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920565376 00000100:00100000:0.0:1767319519.404084:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920565376 00000100:00100000:0.0:1767319519.404135:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4667 00000100:00100000:0.0:1767319519.404144:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920565376:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319519.404266:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920565376:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 121us (329us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319519.404273:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4667 00000100:00100000:0.0:1767319519.404291:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5159 00000100:00100000:0.0:1767319519.404295:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920565248:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319519.404314:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319519.404338:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920565248:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (467us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319519.404344:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5159 00000100:00080000:2.0:1767319519.404909:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319519.963694:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985190400:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319519.963778:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319519.963894:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190400 00000100:00100000:0.0:1767319519.963904:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190400 00000100:00100000:0.0:1767319519.963970:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2236 00000100:00100000:0.0:1767319519.963980:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985190400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319519.964010:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319519.964059:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985190400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (280us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319519.964067:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2236 00000100:00100000:0.0:1767319519.964115:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985190400:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319519.964415:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319519.965576:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985190528:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319519.965629:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319519.965758:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190528 00000100:00100000:2.0:1767319519.965765:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190528 00000100:00080000:3.0:1767319519.965825:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319519.965826:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2237 00000100:00100000:2.0:1767319519.965839:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985190528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319519.965862:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319519.965914:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985190528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (285us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319519.965921:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2237 00000100:00100000:3.0:1767319519.965972:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985190528:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319520.092017:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319520.475537:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319520.475547:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319520.475551:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319520.475555:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319520.475559:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319520.475567:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4763 00000100:00080000:2.0:1767319520.475571:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319520.475577:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319520.475618:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407800 x1853163985190656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319520.475676:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985190656:0@lo:38:kworker.0 00000100:00100000:0.0:1767319520.475742:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319520.475840:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190656 00000100:00100000:1.0:1767319520.475849:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190656 00000100:00100000:1.0:1767319520.475885:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5160 00000100:00100000:1.0:1767319520.475892:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985190656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319520.475912:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319520.475958:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319520.475962:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319520.475965:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319520.475968:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4763 00000100:00080000:2.0:1767319520.475971:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319520.475975:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319520.475990:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578e00 x1853163985190784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319520.477684:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985190784:0@lo:38:kworker.0 00000100:00100000:0.0:1767319520.477748:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319520.477854:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985190656:12345-0@lo:38:kworker.0 Request processed in 1959us (2109us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319520.477864:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5160 00000100:00100000:3.0:1767319520.477966:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190784 00000100:00100000:3.0:1767319520.477974:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190784 00000100:00100000:3.0:1767319520.478013:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1780 00000100:00100000:3.0:1767319520.478019:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985190784:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319520.478039:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319520.478074:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985190784:12345-0@lo:38:kworker.0 Request processed in 54us (326us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319520.478080:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1780 00000100:00100000:3.0:1767319520.478794:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985190912:0@lo:400:kworker.0 00000100:00100000:3.0:1767319520.478830:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319520.478885:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985190912 00000100:00100000:1.0:1767319520.478890:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985190912 00000100:00100000:1.0:1767319520.478937:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4668 00000100:00100000:1.0:1767319520.478946:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985190912:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319520.480032:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319520.480039:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319520.480043:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319520.480049:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4763 00000100:00080000:2.0:1767319520.480053:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319520.480058:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319520.480085:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579f80 x1853163985191040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319520.480124:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319520.480127:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319520.480129:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319520.480133:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4763 00000100:00080000:2.0:1767319520.480135:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319520.480138:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319520.480150:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579500 x1853163985191296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319520.480206:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985191168:0@lo:400:kworker.0 00000100:00100000:2.0:1767319520.480248:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319520.480289:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191168 00000100:00100000:2.0:1767319520.480295:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191168 00000100:00100000:2.0:1767319520.480333:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4669 00000100:00100000:2.0:1767319520.480340:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985191168:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319520.480383:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985191168:12345-0@lo:400:kworker.0 Request processed in 43us (133us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319520.480389:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4669 00000100:00100000:2.0:1767319520.480430:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985191168:0@lo:400:kworker.0 00000100:00080000:2.0:1767319520.480484:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319520.482489:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319520.482499:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319520.482504:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985190656:0@lo:38:kworker.0 00000100:00100000:0.0:1767319520.486312:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985191040:0@lo:38:kworker.0 00000100:00100000:0.0:1767319520.486392:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319520.486438:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985191296:0@lo:38:kworker.0 00000100:00100000:0.0:1767319520.486462:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319520.486506:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191040 00000100:00100000:0.0:1767319520.486514:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191040 00000100:00100000:0.0:1767319520.486548:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191296 00000100:00100000:0.0:1767319520.486551:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191296 00000100:00100000:0.0:1767319520.486577:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5161 00000100:00100000:0.0:1767319520.486584:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985191040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319520.486605:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319520.486641:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985191040:12345-0@lo:38:kworker.0 Request processed in 55us (248us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319520.486647:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5161 00000100:00100000:0.0:1767319520.486664:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5162 00000100:00100000:0.0:1767319520.486668:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985191296:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319520.486677:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319520.486701:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985191296:12345-0@lo:38:kworker.0 Request processed in 31us (238us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319520.486706:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5162 00000100:00080000:0.0:1767319520.486741:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319520.486746:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319520.486749:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985191296:0@lo:38:kworker.0 00000100:00080000:0.0:1767319520.486761:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319520.486764:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319520.486766:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985190784:0@lo:38:kworker.0 00000100:00080000:0.0:1767319520.486783:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319520.486786:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319520.486789:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985191040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319520.490329:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985190912:12345-0@lo:400:kworker.0 Request processed in 11373us (11486us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319520.490344:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4668 00000100:00100000:3.0:1767319520.494747:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985190912:0@lo:400:kworker.0 00010000:00010000:1.0:1767319521.117653:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319522.139935:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319523.164074:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319524.187474:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319524.522077:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319524.522137:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319524.522241:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920565632 00000100:00100000:2.0:1767319524.522252:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920565632 00000100:00100000:2.0:1767319524.522321:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920565504 00000100:00100000:2.0:1767319524.522324:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920565504 00000100:00100000:2.0:1767319524.522340:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4670 00000100:00100000:2.0:1767319524.522351:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920565632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319524.522379:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5163 00000100:00100000:1.0:1767319524.522393:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920565504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319524.522412:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319524.522422:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920565632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 72us (355us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319524.522431:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4670 00000100:00100000:1.0:1767319524.522443:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920565504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 55us (305us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319524.522448:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5163 00000100:00080000:0.0:1767319524.522580:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319525.083546:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319525.083689:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985191552:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319525.083750:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319525.083817:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191552 00000100:00100000:2.0:1767319525.083824:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191552 00000100:00100000:2.0:1767319525.083866:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2238 00000100:00100000:2.0:1767319525.083872:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985191552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319525.083896:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319525.083938:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985191552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (188us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319525.083943:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2238 00000100:00080000:2.0:1767319525.083960:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319525.083981:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985191552:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319525.084642:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985191424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319525.084683:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319525.085877:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191424 00000100:00100000:2.0:1767319525.085890:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191424 00000100:00100000:2.0:1767319525.086021:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2239 00000100:00100000:2.0:1767319525.086031:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985191424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319525.086066:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319525.086440:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985191424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 410us (1755us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319525.086448:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2239 00000100:00100000:1.0:1767319525.087784:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985191424:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319525.087997:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319525.211410:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319525.595301:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319525.595311:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319525.595314:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319525.595318:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319525.595322:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319525.595329:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4768 00000100:00080000:2.0:1767319525.595333:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319525.595339:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319525.595445:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578380 x1853163985191680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319525.595464:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319525.595468:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319525.595470:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319525.595473:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4768 00000100:00080000:2.0:1767319525.595476:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319525.595480:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319525.595496:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985191680:0@lo:38:kworker.0 00000100:00100000:0.0:1767319525.595565:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319525.595580:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a300 x1853163985191808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319525.595618:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319525.595622:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767319525.595663:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985191936:0@lo:400:kworker.0 00000100:00100000:0.0:1767319525.595683:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985191808:0@lo:38:kworker.0 00000100:00100000:3.0:1767319525.595691:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319525.595703:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319525.595710:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319525.595715:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4768 00000100:00080000:2.0:1767319525.595717:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319525.595721:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319525.595733:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578e00 x1853163985192064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319525.595751:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191808 00000100:00100000:0.0:1767319525.595773:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191808 00000100:00100000:0.0:1767319525.595813:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1781 00000100:00100000:0.0:1767319525.595820:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985191808:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767319525.595831:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319525.595834:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319525.595836:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00010000:00020000:0.0:1767319525.595839:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319525.595839:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4768 00000100:00080000:2.0:1767319525.595841:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319525.595844:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319525.595860:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb480 x1853163985192320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319525.595874:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985191808:12345-0@lo:38:kworker.0 Request processed in 53us (170us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319525.595881:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1781 00000100:00100000:0.0:1767319525.595912:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985192192:0@lo:400:kworker.0 00000100:00100000:0.0:1767319525.595939:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985192064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319525.596804:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191680 00000100:00100000:1.0:1767319525.596812:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191680 00000100:00100000:2.0:1767319525.596820:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985191936 00000100:00100000:2.0:1767319525.596824:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985191936 00000100:00100000:1.0:1767319525.596883:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4671 00000100:00100000:1.0:1767319525.596892:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985191936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319525.596943:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985191936:12345-0@lo:400:kworker.0 Request processed in 53us (1251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319525.596950:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4671 00000100:00100000:1.0:1767319525.596964:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5164 00000100:00100000:1.0:1767319525.596968:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985191680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319525.596981:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319525.597007:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985191680:12345-0@lo:38:kworker.0 Request processed in 37us (1442us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319525.597013:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5164 00000100:00100000:3.0:1767319525.597056:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319525.604086:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192192 00000100:00100000:2.0:1767319525.604102:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192192 00000100:00100000:2.0:1767319525.604204:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4672 00000100:00100000:2.0:1767319525.604216:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985192192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319525.604310:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985192192:12345-0@lo:400:kworker.0 Request processed in 97us (7254us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319525.604317:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4672 00000100:00080000:2.0:1767319525.604348:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319525.604465:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985192192:0@lo:400:kworker.0 00000100:00100000:2.0:1767319525.605134:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319525.605189:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985192320:0@lo:38:kworker.0 00000100:00100000:2.0:1767319525.605228:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319525.605254:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319525.605260:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319525.605263:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985191808:0@lo:38:kworker.0 00000100:00080000:2.0:1767319525.605275:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319525.605279:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319525.605281:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985191680:0@lo:38:kworker.0 00000100:00100000:2.0:1767319525.605311:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192064 00000100:00100000:2.0:1767319525.605316:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192064 00000100:00100000:2.0:1767319525.605332:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192320 00000100:00100000:2.0:1767319525.605334:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192320 00000100:00100000:2.0:1767319525.605344:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5165 00000100:00100000:2.0:1767319525.605350:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985192064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319525.605455:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319525.605584:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985192064:12345-0@lo:38:kworker.0 Request processed in 232us (451us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319525.605592:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5165 00000100:00100000:2.0:1767319525.605606:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5166 00000100:00100000:2.0:1767319525.605610:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985192320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319525.605620:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319525.605645:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985192320:12345-0@lo:38:kworker.0 Request processed in 33us (417us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319525.605651:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5166 00000100:00080000:2.0:1767319525.605675:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319525.605680:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319525.605683:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985192064:0@lo:38:kworker.0 00000100:00080000:2.0:1767319525.605691:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319525.605693:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319525.605695:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985192320:0@lo:38:kworker.0 00000100:00100000:2.0:1767319525.605728:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985191936:0@lo:400:kworker.0 00010000:00010000:0.0:1767319526.235641:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319527.259453:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319528.283893:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319529.307527:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319529.647870:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319529.647990:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319529.648284:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920565888 00000100:00100000:1.0:1767319529.648304:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920565888 00000100:00100000:1.0:1767319529.648370:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920565760 00000100:00100000:1.0:1767319529.648374:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920565760 00000100:00100000:1.0:1767319529.648396:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4673 00000100:00100000:1.0:1767319529.648406:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920565888:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319529.648485:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920565888:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (627us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319529.648494:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4673 00000100:00100000:1.0:1767319529.648513:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5167 00000100:00100000:1.0:1767319529.648518:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920565760:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319529.648537:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319529.648569:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920565760:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (579us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319529.648576:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5167 00000100:00080000:2.0:1767319529.648975:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319530.206190:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319530.206212:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985192448:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319530.206273:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319530.207044:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319530.207056:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985192576:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319530.207089:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319530.207662:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192448 00000100:00100000:3.0:1767319530.207672:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192448 00000100:00100000:3.0:1767319530.207735:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192576 00000100:00100000:3.0:1767319530.207738:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192576 00000100:00100000:3.0:1767319530.207761:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2240 00000100:00100000:3.0:1767319530.207770:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985192448:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319530.207799:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319530.207848:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985192448:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (1575us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319530.207856:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2240 00000100:00100000:3.0:1767319530.207873:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2241 00000100:00100000:3.0:1767319530.207879:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985192576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319530.207893:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319530.207921:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985192576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (831us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319530.207927:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2241 00000100:00080000:1.0:1767319530.210081:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319530.210091:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319530.210195:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985192576:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319530.210253:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985192448:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319530.331799:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319530.715660:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319530.715670:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319530.715673:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319530.715678:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319530.715683:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319530.715691:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4773 00000100:00080000:3.0:1767319530.715696:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319530.715702:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319530.715737:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985192704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319530.715842:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319530.715846:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319530.715847:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767319530.715938:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985192704:0@lo:38:kworker.0 00000100:00080000:3.0:1767319530.715977:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4773 00000100:00080000:3.0:1767319530.715981:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319530.715999:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319530.716016:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05c00 x1853163985192832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319530.716049:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319530.716053:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319530.716056:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319530.716069:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4773 00000100:00080000:3.0:1767319530.716073:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319530.716077:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319530.716091:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07480 x1853163985193088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319530.716119:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985192960:0@lo:400:kworker.0 00000100:00080000:3.0:1767319530.716122:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319530.716129:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319530.716131:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319530.716136:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4773 00000100:00080000:3.0:1767319530.716139:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319530.716143:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319530.716181:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985193344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319530.716203:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319530.716213:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985193216:0@lo:400:kworker.0 00000100:00100000:1.0:1767319530.716451:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319530.716516:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192960 00000100:00100000:1.0:1767319530.716523:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192960 00000100:00100000:1.0:1767319530.716562:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4674 00000100:00100000:1.0:1767319530.716578:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985192960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319530.716620:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985192960:12345-0@lo:400:kworker.0 Request processed in 43us (169us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319530.716625:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4674 00000100:00080000:1.0:1767319530.716643:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319530.716669:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985192960:0@lo:400:kworker.0 00000100:00100000:3.0:1767319530.718136:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319530.718188:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319530.718238:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985192832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319530.718266:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319530.718348:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985193088:0@lo:38:kworker.0 00000100:00100000:2.0:1767319530.718365:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193216 00000100:00100000:0.0:1767319530.718371:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319530.718375:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193216 00000100:00100000:0.0:1767319530.718387:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985193344:0@lo:38:kworker.0 00000100:00100000:0.0:1767319530.718407:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319530.718429:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192704 00000100:00100000:0.0:1767319530.718434:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193088 00000100:00100000:2.0:1767319530.718434:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192704 00000100:00100000:0.0:1767319530.718440:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193088 00000100:00100000:2.0:1767319530.718456:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193344 00000100:00100000:0.0:1767319530.718460:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4675 00000100:00100000:2.0:1767319530.718462:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193344 00000100:00100000:0.0:1767319530.718468:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985193216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319530.718483:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5168 00000100:00100000:2.0:1767319530.718491:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985192704:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319530.718505:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985193216:12345-0@lo:400:kworker.0 Request processed in 38us (374us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319530.718511:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4675 00010000:00020000:2.0:1767319530.718512:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319530.718526:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5169 00000100:00100000:0.0:1767319530.718530:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985193088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319530.718538:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319530.718564:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985192704:12345-0@lo:38:kworker.0 Request processed in 71us (374us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319530.718571:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5168 00000100:00100000:0.0:1767319530.718581:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985193088:12345-0@lo:38:kworker.0 Request processed in 48us (209us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319530.718586:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5169 00000100:00100000:3.0:1767319530.718742:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985192832 00000100:00100000:3.0:1767319530.718746:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985192832 00000100:00100000:3.0:1767319530.718775:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1782 00000100:00100000:3.0:1767319530.718781:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985192832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319530.718792:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319530.718818:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985192832:12345-0@lo:38:kworker.0 Request processed in 37us (550us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319530.718824:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1782 00000100:00100000:3.0:1767319530.718860:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985193216:0@lo:400:kworker.0 00000100:00100000:2.0:1767319530.720348:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5170 00000100:00100000:2.0:1767319530.720359:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985193344:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319530.720376:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319530.720452:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319530.720459:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319530.720462:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985192832:0@lo:38:kworker.0 00000100:00080000:0.0:1767319530.720473:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319530.720476:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319530.720479:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985193088:0@lo:38:kworker.0 00000100:00080000:0.0:1767319530.720488:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319530.720491:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319530.720493:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985192704:0@lo:38:kworker.0 00000100:00080000:0.0:1767319530.720547:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319530.720896:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985193344:12345-0@lo:38:kworker.0 Request processed in 536us (2487us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319530.720905:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5170 00000100:00080000:0.0:1767319530.720982:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319530.720999:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319530.721002:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985193344:0@lo:38:kworker.0 00010000:00010000:1.0:1767319531.355378:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319532.379543:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319533.403357:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319534.427413:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319534.767705:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319534.767870:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920566144 00000100:00100000:1.0:1767319534.767881:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920566144 00000100:00100000:1.0:1767319534.767980:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4676 00000100:00100000:1.0:1767319534.767991:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920566144:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319534.768074:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920566144:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (379us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319534.768082:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4676 00000100:00080000:0.0:1767319534.768160:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319534.769356:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319534.769482:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920566016 00000100:00100000:3.0:1767319534.769490:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920566016 00000100:00100000:3.0:1767319534.769531:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5171 00000100:00100000:3.0:1767319534.769537:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920566016:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319534.769558:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319534.769610:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920566016:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 71us (260us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319534.769617:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5171 00000100:00100000:0.0:1767319535.323713:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985193472:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319535.323800:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319535.324214:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193472 00000100:00100000:1.0:1767319535.324223:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193472 00000100:00100000:1.0:1767319535.324289:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2242 00000100:00100000:1.0:1767319535.324298:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985193472:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319535.324326:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319535.324376:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985193472:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (576us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319535.324384:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2242 00000100:00100000:1.0:1767319535.324428:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985193472:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319535.324529:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319535.324676:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985193600:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319535.324707:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319535.324963:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193600 00000100:00100000:2.0:1767319535.324968:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193600 00000100:00100000:2.0:1767319535.325016:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2243 00000100:00100000:2.0:1767319535.325025:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985193600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319535.325041:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319535.325078:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985193600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (370us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319535.325084:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2243 00000100:00100000:2.0:1767319535.325115:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985193600:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319535.326127:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319535.451399:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319535.835503:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319535.835514:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319535.835518:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319535.835522:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319535.835526:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319535.835536:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4778 00000100:00080000:3.0:1767319535.835541:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319535.835549:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319535.835603:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519c00 x1853163985193728/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319535.835631:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319535.835637:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319535.835639:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319535.835642:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4778 00000100:00080000:3.0:1767319535.835652:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319535.835655:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319535.835670:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519500 x1853163985193856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319535.835708:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319535.835712:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319535.835714:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319535.835718:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4778 00000100:00080000:3.0:1767319535.835720:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319535.835723:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319535.835735:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985194112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319535.835778:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319535.835782:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319535.835784:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319535.835787:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4778 00000100:00080000:3.0:1767319535.835790:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319535.835793:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319535.835805:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985194368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319535.835841:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985194240:0@lo:400:kworker.0 00000100:00100000:3.0:1767319535.835942:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319535.836000:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319535.836007:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985194496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319535.836034:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319535.836066:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985194496 00000100:00100000:3.0:1767319535.836074:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985194496 00000100:00100000:3.0:1767319535.836121:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3069 00000100:00100000:3.0:1767319535.836130:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985194496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319535.836201:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985194496:12345-0@lo:400:kworker.0 Request processed in 72us (165us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319535.836209:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3069 00000100:00100000:3.0:1767319535.836253:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985194496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836327:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985193984:0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836355:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319535.836387:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319535.836393:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985194624:0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836414:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319535.836446:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985194240 00000100:00100000:1.0:1767319535.836450:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985194240 00000100:00100000:1.0:1767319535.836483:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193984 00000100:00100000:1.0:1767319535.836487:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193984 00000100:00100000:1.0:1767319535.836506:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4677 00000100:00100000:1.0:1767319535.836512:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985194240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836548:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985194240:12345-0@lo:400:kworker.0 Request processed in 36us (606us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319535.836554:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4677 00000100:00100000:1.0:1767319535.836567:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4678 00000100:00100000:1.0:1767319535.836573:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985193984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836603:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985193984:12345-0@lo:400:kworker.0 Request processed in 31us (247us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319535.836609:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4678 00000100:00100000:1.0:1767319535.836624:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985194624 00000100:00100000:1.0:1767319535.836627:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985194624 00000100:00100000:1.0:1767319535.836644:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3070 00000100:00100000:1.0:1767319535.836650:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985194624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836675:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985194624:12345-0@lo:400:kworker.0 Request processed in 25us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319535.836681:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3070 00000100:00100000:1.0:1767319535.836705:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985194624:0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836714:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985193984:0@lo:400:kworker.0 00000100:00100000:1.0:1767319535.836736:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985194240:0@lo:400:kworker.0 00000100:00080000:0.0:1767319535.837128:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319535.837150:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985193728:0@lo:38:kworker.0 00000100:00100000:0.0:1767319535.837202:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319535.837247:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985193856:0@lo:38:kworker.0 00000100:00100000:0.0:1767319535.837273:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319535.837303:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985194112:0@lo:38:kworker.0 00000100:00100000:0.0:1767319535.837324:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319535.837337:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985194368:0@lo:38:kworker.0 00000100:00100000:0.0:1767319535.837354:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319535.837383:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193728 00000100:00100000:0.0:1767319535.837387:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193728 00000100:00100000:0.0:1767319535.837405:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985194112 00000100:00100000:0.0:1767319535.837408:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985194112 00000100:00100000:0.0:1767319535.837420:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985194368 00000100:00100000:0.0:1767319535.837422:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985194368 00000100:00100000:0.0:1767319535.837432:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5172 00000100:00100000:0.0:1767319535.837437:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985193728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319535.837455:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319535.837487:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985193728:12345-0@lo:38:kworker.0 Request processed in 48us (284us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319535.837493:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5172 00000100:00100000:0.0:1767319535.837507:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5173 00000100:00100000:0.0:1767319535.837511:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985194112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319535.837521:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319535.837542:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985194112:12345-0@lo:38:kworker.0 Request processed in 29us (217us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319535.837547:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5173 00000100:00100000:0.0:1767319535.837558:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5174 00000100:00100000:0.0:1767319535.837562:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985194368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319535.837571:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319535.837592:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985194368:12345-0@lo:38:kworker.0 Request processed in 28us (237us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319535.837598:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5174 00000100:00080000:0.0:1767319535.837632:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319535.837637:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319535.837640:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985193728:0@lo:38:kworker.0 00000100:00080000:0.0:1767319535.837648:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319535.837651:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319535.837654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985194112:0@lo:38:kworker.0 00000100:00080000:0.0:1767319535.837660:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319535.837663:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319535.837665:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985194368:0@lo:38:kworker.0 00000100:00100000:2.0:1767319535.837955:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985193856 00000100:00100000:2.0:1767319535.837961:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985193856 00000100:00100000:2.0:1767319535.837993:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1783 00000100:00100000:2.0:1767319535.837999:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985193856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319535.838010:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319535.838046:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985193856:12345-0@lo:38:kworker.0 Request processed in 46us (771us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319535.838052:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1783 00000100:00080000:2.0:1767319535.838079:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319535.838083:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319535.838086:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985193856:0@lo:38:kworker.0 00010000:00010000:1.0:1767319536.477867:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319537.500750:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319538.523734:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319539.549265:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319539.882681:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319539.882864:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920566528 00000100:00100000:2.0:1767319539.882877:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920566528 00000100:00080000:0.0:1767319539.883051:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319539.883128:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4679 00000100:00100000:2.0:1767319539.883140:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920566528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319539.883253:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920566528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 116us (585us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319539.883261:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4679 00000400:00000100:2.0:1767319539.889088:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920566272 offset 0 length 520: 4 00000100:00100000:2.0:1767319539.889145:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319539.898621:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920566400 00000100:00100000:1.0:1767319539.898639:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920566400 00000100:00100000:1.0:1767319539.898701:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5175 00000100:00100000:1.0:1767319539.898710:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920566400:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319539.898736:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319539.898827:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920566400:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 106us (9672us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319539.898836:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5175 00000100:00100000:2.0:1767319540.443822:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319540.443839:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985194752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319540.443905:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319540.444098:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985194752 00000100:00100000:3.0:1767319540.444107:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985194752 00000100:00080000:0.0:1767319540.444184:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319540.444200:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2244 00000100:00100000:3.0:1767319540.444210:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985194752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319540.444237:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319540.444288:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985194752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (382us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319540.444295:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2244 00000100:00100000:0.0:1767319540.444355:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985194752:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319540.444663:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985194880:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319540.444695:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319540.444751:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985194880 00000100:00100000:2.0:1767319540.444757:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985194880 00000100:00100000:2.0:1767319540.444798:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2245 00000100:00100000:2.0:1767319540.444815:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985194880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319540.444831:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319540.444873:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985194880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (177us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319540.444879:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2245 00000100:00080000:0.0:1767319540.444935:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319540.444944:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985194880:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319540.571362:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319540.956439:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319515/real 1767319515] req@ffff8ce188f05880 x1853163985189504/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319540 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319540.956470:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188f05880 x1853163985189504/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319540 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319540.956493:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319540.956511:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319540.956514:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985189504:0@lo:250:kworker.0 00000100:00080000:3.0:1767319540.956887:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319540.956895:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319540.956898:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319540.956903:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4763 00000100:00080000:3.0:1767319540.956907:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319540.956914:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319540.956947:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631500 x1853163985195008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319540.956972:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319540.956976:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319540.956978:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319540.956981:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4784 00000100:00080000:3.0:1767319540.956983:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319540.956986:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319540.957002:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632d80 x1853163985195136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319540.957011:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319540.957014:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319540.957015:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319540.957018:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4784 00000100:00080000:3.0:1767319540.957020:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319540.957023:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319540.957034:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985195264/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319540.957070:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319540.957074:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319540.957076:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319540.957079:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4784 00000100:00080000:3.0:1767319540.957081:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319540.957084:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319540.957095:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6a00 x1853163985195520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319540.957121:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319540.957124:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319540.957125:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319540.957128:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4784 00000100:00080000:3.0:1767319540.957130:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319540.957132:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319540.957146:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593bb80 x1853163985195776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319540.957201:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985195008:0@lo:250:kworker.0 00000100:00100000:3.0:1767319540.957229:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985195648:0@lo:400:kworker.0 00000400:00000100:0.0:1767319540.957253:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985195008 offset 0 length 520: 4 00000100:00100000:3.0:1767319540.957275:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319540.957299:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985195136:0@lo:38:kworker.0 00000100:00100000:3.0:1767319540.957305:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319540.957313:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985195904:0@lo:400:kworker.0 00000100:00100000:0.0:1767319540.957327:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319540.957347:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319540.957350:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985195264:0@lo:38:kworker.0 00000100:00100000:0.0:1767319540.957383:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319540.957385:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985195904 00000100:00100000:3.0:1767319540.957393:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985195904 00000100:00100000:0.0:1767319540.957406:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985195520:0@lo:38:kworker.0 00000100:00100000:0.0:1767319540.957431:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319540.957445:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3071 00000100:00100000:0.0:1767319540.957451:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985195776:0@lo:38:kworker.0 00000100:00100000:3.0:1767319540.957458:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985195904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319540.957476:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319540.957516:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985195264 00000100:00100000:3.0:1767319540.957517:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985195904:12345-0@lo:400:kworker.0 Request processed in 60us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319540.957523:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985195264 00000100:00100000:3.0:1767319540.957527:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3071 00000100:00100000:0.0:1767319540.957551:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1784 00000100:00100000:0.0:1767319540.957558:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985195264:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319540.957566:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985195904:0@lo:400:kworker.0 00010000:00020000:0.0:1767319540.957577:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319540.957599:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985195648 00000100:00100000:3.0:1767319540.957604:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985195648 00000100:00100000:0.0:1767319540.957616:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985195264:12345-0@lo:38:kworker.0 Request processed in 55us (231us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319540.957625:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1784 00000100:00100000:3.0:1767319540.957640:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985195136 00000100:00100000:3.0:1767319540.957643:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985195136 00000100:00080000:0.0:1767319540.957647:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319540.957657:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985195520 00000100:00100000:3.0:1767319540.957660:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985195520 00000100:00100000:0.0:1767319540.957664:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985195776 00000100:00100000:0.0:1767319540.957670:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985195776 00000100:00100000:3.0:1767319540.957673:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4680 00000100:00100000:3.0:1767319540.957682:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985195648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319540.957692:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5176 00000100:00100000:0.0:1767319540.957699:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985195136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319540.957710:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319540.957717:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985195648:12345-0@lo:400:kworker.0 Request processed in 36us (442us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319540.957724:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4680 00000100:00100000:3.0:1767319540.957740:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5177 00000100:00100000:0.0:1767319540.957745:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985195136:12345-0@lo:38:kworker.0 Request processed in 44us (418us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319540.957747:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985195520:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319540.957759:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5176 00010000:00020000:3.0:1767319540.957763:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319540.957777:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5178 00000100:00100000:0.0:1767319540.957783:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985195776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319540.957794:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319540.957794:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985195520:12345-0@lo:38:kworker.0 Request processed in 46us (362us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319540.957815:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5177 00000100:00100000:0.0:1767319540.957837:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985195776:12345-0@lo:38:kworker.0 Request processed in 51us (360us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319540.957844:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5178 00000100:00100000:3.0:1767319540.957848:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985195648:0@lo:400:kworker.0 00000100:00080000:0.0:1767319540.957879:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319540.957884:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319540.957888:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985195136:0@lo:38:kworker.0 00000100:00080000:0.0:1767319540.957896:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319540.957899:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319540.957902:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985195264:0@lo:38:kworker.0 00000100:00080000:0.0:1767319540.957909:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319540.957912:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319540.957915:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985195520:0@lo:38:kworker.0 00000100:00080000:0.0:1767319540.957921:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319540.957924:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319540.957928:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985195776:0@lo:38:kworker.0 00000100:00100000:1.0:1767319540.958013:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985195392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319540.958041:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319540.958070:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319540.958075:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985196032:0@lo:400:kworker.0 00000100:00100000:1.0:1767319540.958106:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319540.958141:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196032 00000100:00100000:1.0:1767319540.958145:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196032 00000100:00100000:1.0:1767319540.958199:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3072 00000100:00100000:1.0:1767319540.958208:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985196032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319540.958230:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985195392 00000100:00100000:3.0:1767319540.958235:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985195392 00000100:00100000:1.0:1767319540.958249:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985196032:12345-0@lo:400:kworker.0 Request processed in 42us (141us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319540.958257:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3072 00000100:00100000:3.0:1767319540.958265:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4681 00000100:00100000:3.0:1767319540.958273:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985195392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319540.958296:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985196032:0@lo:400:kworker.0 00000100:00100000:3.0:1767319540.958307:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985195392:12345-0@lo:400:kworker.0 Request processed in 35us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319540.958313:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985195392:0@lo:400:kworker.0 00000100:00100000:3.0:1767319540.958317:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4681 00000100:00080000:0.0:1767319540.958459:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319541.595475:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319542.622357:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319543.643645:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319544.670357:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319545.003549:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319545.003645:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920566784 00000100:00100000:2.0:1767319545.003662:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920566784 00000100:00100000:2.0:1767319545.003771:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4682 00000100:00100000:2.0:1767319545.003791:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920566784:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767319545.003894:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319545.003906:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920566784:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 124us (370us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319545.003915:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4682 00000100:00100000:3.0:1767319545.005509:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319545.005597:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920566656 00000100:00100000:2.0:1767319545.005611:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920566656 00000100:00100000:2.0:1767319545.005663:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5179 00000100:00100000:2.0:1767319545.005670:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920566656:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319545.005693:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319545.005770:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920566656:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 99us (289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319545.005776:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5179 00000100:00100000:2.0:1767319545.564852:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319545.564879:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985196160:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319545.564957:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319545.565182:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319545.565193:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985196288:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319545.565226:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319545.565314:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196160 00000100:00100000:3.0:1767319545.565325:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196160 00000100:00100000:3.0:1767319545.565405:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196288 00000100:00100000:3.0:1767319545.565410:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196288 00000100:00100000:3.0:1767319545.565434:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2246 00000100:00100000:3.0:1767319545.565442:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985196160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319545.565472:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319545.565519:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985196160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (561us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319545.565526:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2246 00000100:00100000:3.0:1767319545.565542:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2247 00000100:00100000:3.0:1767319545.565549:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985196288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319545.565574:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319545.565602:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985196288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (376us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319545.565608:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2247 00000100:00100000:3.0:1767319545.565649:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985196160:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319545.565825:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319545.565828:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319545.565852:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985196288:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319545.695603:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319546.076185:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319546.076195:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319546.076198:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319546.076202:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319546.076207:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319546.076215:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4789 00000100:00080000:3.0:1767319546.076219:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319546.076224:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319546.076267:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593b800 x1853163985196416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319546.076299:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319546.076303:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319546.076305:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319546.076309:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4789 00000100:00080000:3.0:1767319546.076311:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319546.076314:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319546.076328:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939880 x1853163985196544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319546.076368:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319546.076371:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319546.076373:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319546.076377:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4789 00000100:00080000:3.0:1767319546.076379:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319546.076382:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319546.076400:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5f80 x1853163985196800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319546.076427:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319546.076431:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319546.076433:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319546.076437:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4789 00000100:00080000:3.0:1767319546.076439:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319546.076442:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319546.076458:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985197056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319546.076523:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985196928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319546.076587:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319546.076646:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319546.076653:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985197184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319546.076678:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319546.076787:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196928 00000100:00100000:0.0:1767319546.076794:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196928 00000100:00100000:0.0:1767319546.076859:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4683 00000100:00100000:0.0:1767319546.076868:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985196928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319546.076919:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985196928:12345-0@lo:400:kworker.0 Request processed in 53us (331us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319546.076926:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4683 00000100:00100000:0.0:1767319546.076953:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985196416:0@lo:38:kworker.0 00000100:00100000:0.0:1767319546.076976:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319546.076995:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985196544:0@lo:38:kworker.0 00000100:00100000:0.0:1767319546.077016:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319546.077037:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985196800:0@lo:38:kworker.0 00000100:00100000:0.0:1767319546.077056:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319546.077068:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985197056:0@lo:38:kworker.0 00000100:00100000:0.0:1767319546.077084:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319546.077111:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196544 00000100:00100000:0.0:1767319546.077115:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196544 00000100:00100000:0.0:1767319546.077134:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1785 00000100:00100000:0.0:1767319546.077138:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985196544:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319546.077155:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319546.077208:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985196544:12345-0@lo:38:kworker.0 Request processed in 68us (191us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319546.077214:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1785 00000100:00100000:0.0:1767319546.077230:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196416 00000100:00100000:0.0:1767319546.077234:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196416 00000100:00100000:0.0:1767319546.077249:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196800 00000100:00100000:0.0:1767319546.077252:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196800 00000100:00100000:0.0:1767319546.077262:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197056 00000100:00100000:0.0:1767319546.077264:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197056 00000100:00100000:0.0:1767319546.077275:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5180 00000100:00100000:0.0:1767319546.077279:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985196416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319546.077288:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319546.077312:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985196416:12345-0@lo:38:kworker.0 Request processed in 31us (335us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319546.077317:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5180 00000100:00100000:0.0:1767319546.077328:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5181 00000100:00100000:0.0:1767319546.077332:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985196800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319546.077341:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319546.077362:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985196800:12345-0@lo:38:kworker.0 Request processed in 29us (306us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319546.077368:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5181 00000100:00100000:0.0:1767319546.077378:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5182 00000100:00100000:0.0:1767319546.077382:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985197056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319546.077390:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319546.077410:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985197056:12345-0@lo:38:kworker.0 Request processed in 27us (324us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319546.077416:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5182 00000100:00080000:0.0:1767319546.077456:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319546.077461:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319546.077465:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985196416:0@lo:38:kworker.0 00000100:00080000:0.0:1767319546.077474:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319546.077476:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319546.077479:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985196544:0@lo:38:kworker.0 00000100:00080000:0.0:1767319546.077485:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319546.077489:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319546.077491:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985196800:0@lo:38:kworker.0 00000100:00080000:0.0:1767319546.077497:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319546.077499:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319546.077502:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985197056:0@lo:38:kworker.0 00000100:00080000:3.0:1767319546.077582:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319546.077614:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985196928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319546.077689:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197184 00000100:00100000:1.0:1767319546.077694:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197184 00000100:00100000:1.0:1767319546.077747:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3073 00000100:00100000:1.0:1767319546.077754:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985197184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319546.077798:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985197184:12345-0@lo:400:kworker.0 Request processed in 45us (1118us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319546.077804:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3073 00000100:00100000:1.0:1767319546.077826:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985196672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319546.077851:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319546.077869:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319546.077875:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985197312:0@lo:400:kworker.0 00000100:00100000:1.0:1767319546.077895:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319546.077916:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197312 00000100:00100000:1.0:1767319546.077919:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197312 00000100:00100000:1.0:1767319546.077940:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3074 00000100:00100000:1.0:1767319546.077945:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985197312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319546.077978:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985197312:12345-0@lo:400:kworker.0 Request processed in 32us (81us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319546.077984:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3074 00000100:00100000:1.0:1767319546.078019:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985197312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319546.080127:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985196672 00000100:00100000:3.0:1767319546.080138:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985196672 00000100:00100000:3.0:1767319546.080226:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4684 00000100:00100000:3.0:1767319546.080233:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985196672:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319546.080303:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985196672:12345-0@lo:400:kworker.0 Request processed in 71us (2451us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319546.080308:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4684 00000100:00100000:3.0:1767319546.080349:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985197184:0@lo:400:kworker.0 00000100:00080000:1.0:1767319546.080725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319546.080752:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985196672:0@lo:400:kworker.0 00010000:00010000:1.0:1767319546.718644:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319547.741367:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319548.770293:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319549.790355:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319550.121764:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319550.121815:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319550.121888:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920566912 00000100:00100000:3.0:1767319550.121898:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920566912 00000100:00100000:3.0:1767319550.121939:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920567040 00000100:00100000:3.0:1767319550.121942:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920567040 00000100:00100000:3.0:1767319550.121991:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4685 00000100:00100000:3.0:1767319550.122002:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920567040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319550.122079:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920567040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (262us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319550.122087:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4685 00000100:00100000:3.0:1767319550.122106:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5183 00000100:00100000:3.0:1767319550.122110:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920566912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319550.122130:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319550.122155:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920566912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (402us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319550.122197:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5183 00000100:00080000:1.0:1767319550.122483:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319550.684675:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319550.684701:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985197440:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319550.684775:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319550.684918:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197440 00000100:00100000:0.0:1767319550.684927:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197440 00000100:00100000:0.0:1767319550.684997:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2248 00000100:00100000:0.0:1767319550.685007:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985197440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319550.685036:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319550.685084:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985197440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (308us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319550.685092:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2248 00000100:00100000:0.0:1767319550.685141:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985197440:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319550.685258:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319550.688849:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319550.688872:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985197568:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319550.688936:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319550.689047:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197568 00000100:00100000:2.0:1767319550.689056:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197568 00000100:00100000:2.0:1767319550.689123:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2249 00000100:00100000:2.0:1767319550.689134:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985197568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319550.689188:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319550.689228:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985197568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319550.689235:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2249 00000100:00080000:1.0:1767319550.689325:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319550.689357:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985197568:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319550.811456:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319551.196150:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319551.196161:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319551.196179:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319551.196183:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319551.196188:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319551.196196:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4794 00000100:00080000:3.0:1767319551.196201:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319551.196208:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319551.196249:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2a00 x1853163985197696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319551.196279:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319551.196283:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319551.196285:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319551.196289:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4794 00000100:00080000:3.0:1767319551.196291:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319551.196295:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319551.196314:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1f80 x1853163985197824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319551.196394:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985197952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319551.196460:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319551.196583:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197952 00000100:00100000:2.0:1767319551.196592:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197952 00000100:00080000:3.0:1767319551.196660:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319551.196664:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319551.196666:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319551.196671:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4794 00000100:00080000:3.0:1767319551.196673:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319551.196677:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319551.196678:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4686 00000100:00100000:2.0:1767319551.196689:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985197952:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319551.196691:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3800 x1853163985198080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319551.196715:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319551.196718:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319551.196720:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319551.196723:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4794 00000100:00080000:3.0:1767319551.196726:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319551.196728:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319551.196739:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3b80 x1853163985198336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319551.196782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985197696:0@lo:38:kworker.0 00000100:00100000:0.0:1767319551.196812:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319551.196830:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985197824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319551.196853:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319551.196874:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985198080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319551.196911:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319551.196943:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197696 00000100:00100000:0.0:1767319551.196947:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197696 00000100:00100000:0.0:1767319551.197354:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198080 00000100:00100000:0.0:1767319551.197359:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198080 00000100:00100000:0.0:1767319551.197380:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5184 00000100:00100000:0.0:1767319551.197386:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985197696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319551.197404:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319551.197827:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985197696:12345-0@lo:38:kworker.0 Request processed in 437us (1011us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319551.197835:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5184 00000100:00100000:0.0:1767319551.197854:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5185 00000100:00100000:0.0:1767319551.197859:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985198080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319551.197870:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319551.197914:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985198080:12345-0@lo:38:kworker.0 Request processed in 53us (1002us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319551.197920:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5185 00000100:00100000:0.0:1767319551.197942:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985197824 00000100:00100000:0.0:1767319551.197946:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985197824 00000100:00100000:0.0:1767319551.197977:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1786 00000100:00100000:0.0:1767319551.197982:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985197824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319551.197992:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319551.198018:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985197824:12345-0@lo:38:kworker.0 Request processed in 34us (1164us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319551.198026:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1786 00000100:00080000:0.0:1767319551.198066:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319551.198092:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985198208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198116:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319551.198139:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319551.198157:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985198464:0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198202:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319551.198236:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985198592:0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198257:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319551.198286:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198464 00000100:00100000:0.0:1767319551.198290:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198464 00000100:00100000:0.0:1767319551.198316:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198592 00000100:00100000:0.0:1767319551.198319:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198592 00000100:00100000:0.0:1767319551.198336:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3075 00000100:00100000:0.0:1767319551.198343:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985198464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198382:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985198464:12345-0@lo:400:kworker.0 Request processed in 40us (179us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319551.198389:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3075 00000100:00100000:0.0:1767319551.198401:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3076 00000100:00100000:0.0:1767319551.198407:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985198592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198436:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985198592:12345-0@lo:400:kworker.0 Request processed in 30us (177us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319551.198442:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3076 00000100:00100000:0.0:1767319551.198457:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198208 00000100:00100000:0.0:1767319551.198460:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198208 00000100:00100000:0.0:1767319551.198481:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4687 00000100:00100000:0.0:1767319551.198486:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985198208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198510:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985198208:12345-0@lo:400:kworker.0 Request processed in 23us (393us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319551.198516:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4687 00000100:00100000:0.0:1767319551.198694:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985198592:0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198719:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985198464:0@lo:400:kworker.0 00000100:00100000:0.0:1767319551.198727:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985198208:0@lo:400:kworker.0 00000100:00080000:0.0:1767319551.198740:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319551.198772:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985198336:0@lo:38:kworker.0 00000100:00100000:0.0:1767319551.198799:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319551.198823:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319551.198828:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319551.198831:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985197696:0@lo:38:kworker.0 00000100:00080000:0.0:1767319551.198846:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319551.198850:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319551.198853:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985197824:0@lo:38:kworker.0 00000100:00080000:0.0:1767319551.198862:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319551.198865:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319551.198868:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985198080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319551.198941:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198336 00000100:00100000:0.0:1767319551.198945:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198336 00000100:00100000:0.0:1767319551.198964:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5186 00000100:00100000:0.0:1767319551.198968:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985198336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319551.198981:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319551.199009:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985198336:12345-0@lo:38:kworker.0 Request processed in 39us (209us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319551.199014:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5186 00000100:00080000:0.0:1767319551.199036:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319551.199039:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319551.199042:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985198336:0@lo:38:kworker.0 00000100:00100000:2.0:1767319551.200804:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985197952:12345-0@lo:400:kworker.0 Request processed in 4110us (4334us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319551.200814:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4686 00000100:00100000:1.0:1767319551.200888:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985197952:0@lo:400:kworker.0 00010000:00010000:1.0:1767319551.835362:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319552.859405:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319553.884611:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319554.907569:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319555.245004:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319555.245076:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319555.245160:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920567296 00000100:00100000:2.0:1767319555.245200:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920567296 00000100:00100000:2.0:1767319555.245270:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920567168 00000100:00100000:2.0:1767319555.245274:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920567168 00000100:00100000:2.0:1767319555.245290:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4688 00000100:00100000:2.0:1767319555.245301:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920567296:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319555.245373:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920567296:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319555.245392:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4688 00000100:00100000:2.0:1767319555.245411:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5187 00000100:00100000:2.0:1767319555.245415:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920567168:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319555.245433:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319555.245457:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920567168:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (381us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319555.245464:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5187 00000100:00080000:1.0:1767319555.246364:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319555.803395:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319555.803420:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985198720:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319555.803497:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319555.803715:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198720 00000100:00100000:3.0:1767319555.803725:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198720 00000100:00100000:3.0:1767319555.803798:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2250 00000100:00100000:3.0:1767319555.803809:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985198720:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319555.803840:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319555.803882:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985198720:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (385us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319555.803889:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2250 00000100:00080000:0.0:1767319555.803987:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319555.804023:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985198720:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319555.805922:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319555.805942:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985198848:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319555.806001:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319555.806104:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198848 00000100:00100000:2.0:1767319555.806113:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198848 00000100:00100000:2.0:1767319555.806190:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2251 00000100:00100000:2.0:1767319555.806213:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985198848:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319555.806242:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319555.806288:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985198848:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (286us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319555.806296:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2251 00000100:00100000:2.0:1767319555.806343:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985198848:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319555.806508:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319555.931402:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319556.315343:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319556.315354:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319556.315365:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319556.315369:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319556.315373:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319556.315381:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4799 00000100:00080000:1.0:1767319556.315386:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319556.315394:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319556.315435:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc54000 x1853163985198976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319556.315463:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319556.315467:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319556.315469:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319556.315472:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4799 00000100:00080000:1.0:1767319556.315474:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319556.315477:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319556.315493:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc55880 x1853163985199104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319556.315559:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985198976:0@lo:38:kworker.0 00000100:00080000:1.0:1767319556.315568:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319556.315573:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319556.315574:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319556.315578:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4799 00000100:00080000:1.0:1767319556.315581:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319556.315585:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319556.315598:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc56680 x1853163985199360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319556.315631:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319556.315632:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319556.315638:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319556.315640:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319556.315645:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4799 00000100:00080000:1.0:1767319556.315647:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319556.315651:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319556.315662:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985199488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319556.315684:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985199104:0@lo:38:kworker.0 00000100:00100000:1.0:1767319556.315700:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319556.315710:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319556.315740:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985199360:0@lo:38:kworker.0 00000100:00100000:1.0:1767319556.315750:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985198976 00000100:00100000:2.0:1767319556.315755:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319556.315759:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985198976 00000100:00100000:1.0:1767319556.315797:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985199488 00000100:00100000:2.0:1767319556.315799:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985199360 00000100:00100000:1.0:1767319556.315801:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985199488 00000100:00100000:2.0:1767319556.315804:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985199360 00000100:00100000:2.0:1767319556.315826:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5188 00000100:00100000:2.0:1767319556.315832:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985198976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319556.315851:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319556.315854:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4689 00000100:00100000:1.0:1767319556.315866:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985199488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319556.315892:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985198976:12345-0@lo:38:kworker.0 Request processed in 57us (259us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319556.315900:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5188 00000100:00100000:1.0:1767319556.315919:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985199488:12345-0@lo:400:kworker.0 Request processed in 55us (218us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319556.315920:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5189 00000100:00100000:2.0:1767319556.315926:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985199360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319556.315935:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319556.315955:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985199360:12345-0@lo:38:kworker.0 Request processed in 29us (200us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319556.315959:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5189 00000100:00080000:2.0:1767319556.315985:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319556.315988:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319556.315990:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985199360:0@lo:38:kworker.0 00000100:00080000:2.0:1767319556.315995:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319556.315997:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319556.315999:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985198976:0@lo:38:kworker.0 00000100:00080000:2.0:1767319556.316022:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319556.316263:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985199232:0@lo:400:kworker.0 00000100:00100000:0.0:1767319556.316292:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319556.316332:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985199232 00000100:00100000:0.0:1767319556.316336:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985199232 00000100:00100000:0.0:1767319556.316367:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4690 00000100:00100000:0.0:1767319556.316373:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985199232:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319556.316406:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985199232:12345-0@lo:400:kworker.0 Request processed in 34us (113us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319556.316413:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4690 00000100:00100000:0.0:1767319556.316433:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985199104 00000100:00100000:0.0:1767319556.316437:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985199104 00000100:00100000:0.0:1767319556.316458:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1787 00000100:00100000:0.0:1767319556.316464:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985199104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319556.316476:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319556.316511:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985199104:12345-0@lo:38:kworker.0 Request processed in 46us (800us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319556.316525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319556.316532:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1787 00000100:00080000:2.0:1767319556.316555:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319556.316561:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319556.316565:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985199104:0@lo:38:kworker.0 00000100:00100000:0.0:1767319556.316569:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985199232:0@lo:400:kworker.0 00000100:00100000:2.0:1767319556.316606:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985199488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319556.316647:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18dc00 x1853163985199616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319556.316725:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985199616:0@lo:38:kworker.0 00000100:00100000:0.0:1767319556.316744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319556.316806:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985199616 00000100:00100000:3.0:1767319556.316809:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985199616 00000100:00100000:3.0:1767319556.316830:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5190 00000100:00100000:3.0:1767319556.316833:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985199616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319556.316841:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319556.316873:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985199616:12345-0@lo:38:kworker.0 Request processed in 38us (127us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319556.316879:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5190 00000100:00080000:0.0:1767319556.316972:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319556.316977:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319556.316980:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985199616:0@lo:38:kworker.0 00000100:00100000:1.0:1767319556.317877:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4689 00010000:00010000:0.0:1767319556.955523:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319557.979465:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319559.004135:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319560.031583:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319560.362366:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319560.362665:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920567424 00000100:00100000:1.0:1767319560.362676:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920567424 00000100:00100000:1.0:1767319560.363001:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5191 00000100:00100000:1.0:1767319560.363011:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920567424:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319560.363037:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319560.363699:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920567424:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 683us (1338us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319560.363708:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5191 00000100:00100000:3.0:1767319560.364431:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319560.364588:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920567552 00000100:00100000:1.0:1767319560.364598:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920567552 00000100:00100000:1.0:1767319560.364667:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4691 00000100:00100000:1.0:1767319560.364677:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920567552:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319560.364753:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920567552:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (325us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319560.364761:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4691 00000100:00080000:3.0:1767319560.364906:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319560.923581:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319560.923614:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985199744:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319560.923680:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319560.924038:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985199744 00000100:00100000:0.0:1767319560.924047:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985199744 00000100:00100000:0.0:1767319560.924118:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2252 00000100:00100000:0.0:1767319560.924128:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985199744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319560.924157:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319560.924214:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985199744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (533us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319560.924221:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2252 00000100:00100000:1.0:1767319560.924315:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985199744:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319560.924376:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319560.924740:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985199872:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319560.924771:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319560.924866:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985199872 00000100:00100000:2.0:1767319560.924871:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985199872 00000100:00100000:2.0:1767319560.924916:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2253 00000100:00100000:2.0:1767319560.924924:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985199872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319560.924941:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319560.924976:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985199872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (204us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319560.924983:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2253 00000100:00100000:2.0:1767319560.925012:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985199872:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319560.926323:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319561.051619:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319561.435536:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319561.435548:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319561.435551:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319561.435556:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319561.435560:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319561.435569:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4804 00000100:00080000:3.0:1767319561.435574:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319561.435581:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319561.435621:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3100 x1853163985200000/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319561.435645:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319561.435648:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319561.435650:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319561.435654:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4804 00000100:00080000:3.0:1767319561.435656:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319561.435660:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319561.435673:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2680 x1853163985200128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319561.435725:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319561.435729:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319561.435731:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319561.435734:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4804 00000100:00080000:3.0:1767319561.435736:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319561.435739:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319561.435750:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1180 x1853163985200384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319561.435775:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319561.435779:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319561.435781:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319561.435784:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4804 00000100:00080000:3.0:1767319561.435786:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319561.435789:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319561.435801:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3b80 x1853163985200640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319561.435820:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319561.435832:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985200512:0@lo:400:kworker.0 00000100:00100000:3.0:1767319561.435891:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319561.436011:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319561.436021:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985200256:0@lo:400:kworker.0 00000100:00100000:1.0:1767319561.436049:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319561.436267:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985200512 00000100:00100000:0.0:1767319561.436525:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985200256 00000100:00100000:0.0:1767319561.436531:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985200256 00000100:00100000:0.0:1767319561.436588:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4692 00000100:00100000:0.0:1767319561.436600:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985200256:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319561.436658:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985200256:12345-0@lo:400:kworker.0 Request processed in 61us (607us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319561.436665:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4692 00000100:00080000:0.0:1767319561.436691:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319561.436877:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985200256:0@lo:400:kworker.0 00000100:00100000:3.0:1767319561.436931:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985200512 00000100:00100000:2.0:1767319561.436954:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985200000:0@lo:38:kworker.0 00000100:00100000:3.0:1767319561.436978:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4693 00000100:00100000:3.0:1767319561.436987:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985200512:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319561.437011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319561.437043:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985200512:12345-0@lo:400:kworker.0 Request processed in 56us (1149us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319561.437049:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985200128:0@lo:38:kworker.0 00000100:00100000:3.0:1767319561.437055:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4693 00000100:00100000:3.0:1767319561.437071:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985200000 00000100:00100000:3.0:1767319561.437075:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985200000 00000100:00100000:2.0:1767319561.438007:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319561.438051:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985200384:0@lo:38:kworker.0 00000100:00100000:3.0:1767319561.438218:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5192 00000100:00100000:3.0:1767319561.438224:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985200000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319561.438246:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319561.438429:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985200128 00000100:00100000:1.0:1767319561.438435:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985200128 00000100:00100000:1.0:1767319561.438459:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1788 00000100:00100000:1.0:1767319561.438463:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985200128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319561.438474:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319561.438505:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985200128:12345-0@lo:38:kworker.0 Request processed in 40us (510us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319561.438511:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1788 00000100:00100000:1.0:1767319561.438569:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985200512:0@lo:400:kworker.0 00000100:00080000:0.0:1767319561.438661:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319561.438699:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985200000:12345-0@lo:38:kworker.0 Request processed in 474us (1700us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319561.438705:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5192 00000100:00100000:2.0:1767319561.439449:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319561.439493:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985200640:0@lo:38:kworker.0 00000100:00100000:2.0:1767319561.439530:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319561.439578:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319561.439586:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319561.439591:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985200000:0@lo:38:kworker.0 00000100:00100000:1.0:1767319561.439598:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985200384 00000100:00100000:1.0:1767319561.439606:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985200384 00000100:00100000:1.0:1767319561.439641:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985200640 00000100:00100000:1.0:1767319561.439643:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985200640 00000100:00100000:1.0:1767319561.439664:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5193 00000100:00100000:1.0:1767319561.439670:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985200384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319561.439686:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319561.439723:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985200384:12345-0@lo:38:kworker.0 Request processed in 52us (276us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319561.439729:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5193 00000100:00100000:1.0:1767319561.439742:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5194 00000100:00100000:1.0:1767319561.439746:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985200640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319561.439755:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319561.439789:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985200640:12345-0@lo:38:kworker.0 Request processed in 41us (258us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319561.439795:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5194 00000100:00080000:2.0:1767319561.441405:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319561.441416:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319561.442427:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985200128:0@lo:38:kworker.0 00000100:00080000:2.0:1767319561.442462:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319561.442466:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319561.442470:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985200384:0@lo:38:kworker.0 00000100:00080000:2.0:1767319561.442481:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319561.442484:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319561.442487:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985200640:0@lo:38:kworker.0 00010000:00010000:1.0:1767319562.076110:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319563.102331:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319564.123761:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319564.458472:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319564.458710:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920567936 00000100:00100000:3.0:1767319564.458721:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920567936 00000100:00100000:3.0:1767319564.458778:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4694 00000100:00100000:3.0:1767319564.458785:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920567936:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319564.458847:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920567936:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 62us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319564.458855:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4694 00000400:00000100:3.0:1767319564.458931:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920567680 offset 0 length 520: 4 00000100:00100000:3.0:1767319564.459049:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319564.459063:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920567808 00000100:00100000:3.0:1767319564.459066:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920567808 00000100:00100000:3.0:1767319564.459078:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5195 00000100:00100000:3.0:1767319564.459082:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920567808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319564.459097:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319564.459116:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920567808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 33us (68us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319564.459122:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5195 00000100:00080000:2.0:1767319564.460191:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319565.147534:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319565.531659:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319540/real 1767319540] req@ffff8ce188631500 x1853163985195008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319565 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319565.531695:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188631500 x1853163985195008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319565 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319565.531711:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319565.531735:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319565.531739:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985195008:0@lo:250:kworker.0 00000100:00080000:0.0:1767319565.531899:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319565.531906:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319565.531909:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319565.531913:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4789 00000100:00080000:0.0:1767319565.531918:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319565.531926:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319565.531976:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d180 x1853163985200768/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319565.531999:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319565.532003:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319565.532005:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319565.532009:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4809 00000100:00080000:0.0:1767319565.532011:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319565.532014:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319565.532029:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c000 x1853163985200896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319565.532038:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319565.532041:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319565.532043:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319565.532046:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4809 00000100:00080000:0.0:1767319565.532048:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319565.532051:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319565.532064:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4a80 x1853163985201024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319565.532111:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319565.532115:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319565.532117:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319565.532120:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4809 00000100:00080000:0.0:1767319565.532123:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319565.532126:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319565.532137:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985201280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319565.532181:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319565.532185:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319565.532187:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319565.532190:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4809 00000100:00080000:0.0:1767319565.532203:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319565.532207:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319565.532221:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593bb80 x1853163985201536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319565.532261:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985201664:0@lo:400:kworker.0 00000100:00100000:0.0:1767319565.532365:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319565.532429:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319565.532436:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985201408:0@lo:400:kworker.0 00000100:00100000:0.0:1767319565.532461:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319565.532493:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201664 00000100:00100000:0.0:1767319565.532503:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201664 00000100:00100000:3.0:1767319565.533154:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985201792:0@lo:400:kworker.0 00000100:00100000:3.0:1767319565.533221:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319565.533246:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319565.533252:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985201152:0@lo:400:kworker.0 00000100:00100000:3.0:1767319565.533274:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319565.533309:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201408 00000100:00100000:3.0:1767319565.533314:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201408 00000100:00100000:3.0:1767319565.533371:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201152 00000100:00100000:3.0:1767319565.533374:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201152 00000100:00100000:3.0:1767319565.533399:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4695 00000100:00100000:3.0:1767319565.533406:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985201408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319565.533461:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985201408:12345-0@lo:400:kworker.0 Request processed in 54us (998us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319565.533467:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4695 00000100:00100000:3.0:1767319565.533484:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4696 00000100:00100000:3.0:1767319565.533489:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985201152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319565.533530:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985201152:12345-0@lo:400:kworker.0 Request processed in 40us (254us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319565.533536:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4696 00000100:00080000:3.0:1767319565.533554:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319565.533583:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985201152:0@lo:400:kworker.0 00000100:00100000:3.0:1767319565.533605:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985201408:0@lo:400:kworker.0 00000100:00100000:3.0:1767319565.533663:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201792 00000100:00100000:3.0:1767319565.533668:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201792 00000100:00100000:3.0:1767319565.533700:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3077 00000100:00100000:3.0:1767319565.533707:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985201792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319565.533742:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985201792:12345-0@lo:400:kworker.0 Request processed in 36us (521us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319565.533748:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3077 00000100:00080000:3.0:1767319565.533762:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319565.533775:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985201792:0@lo:400:kworker.0 00000100:00100000:0.0:1767319565.534220:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3078 00000100:00100000:0.0:1767319565.534228:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985201664:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319565.534265:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985201664:12345-0@lo:400:kworker.0 Request processed in 38us (1901us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319565.534271:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3078 00000100:00100000:0.0:1767319565.534298:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985201664:0@lo:400:kworker.0 00000100:00100000:2.0:1767319565.534494:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985200768:0@lo:250:kworker.0 00000400:00000100:2.0:1767319565.534840:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985200768 offset 0 length 520: 4 00000100:00100000:2.0:1767319565.534869:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985200896:0@lo:38:kworker.0 00000100:00100000:2.0:1767319565.534897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319565.534922:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985201024:0@lo:38:kworker.0 00000100:00100000:2.0:1767319565.534947:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319565.535106:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985200896 00000100:00100000:1.0:1767319565.535112:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985200896 00000100:00100000:1.0:1767319565.535149:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5196 00000100:00100000:1.0:1767319565.535154:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985200896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319565.535204:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319565.535243:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985200896:12345-0@lo:38:kworker.0 Request processed in 88us (346us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319565.535250:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5196 00000100:00100000:2.0:1767319565.535392:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985201280:0@lo:38:kworker.0 00000100:00100000:2.0:1767319565.535416:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319565.535434:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985201536:0@lo:38:kworker.0 00000100:00100000:2.0:1767319565.535455:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319565.535485:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319565.535491:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319565.535495:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985200896:0@lo:38:kworker.0 00000100:00100000:1.0:1767319565.535602:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201024 00000100:00100000:1.0:1767319565.535607:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201024 00000100:00100000:1.0:1767319565.535642:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1789 00000100:00100000:1.0:1767319565.535647:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985201024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319565.535659:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319565.535724:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201280 00000100:00100000:0.0:1767319565.535728:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201280 00000100:00100000:0.0:1767319565.535748:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201536 00000100:00100000:0.0:1767319565.535751:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201536 00000100:00080000:3.0:1767319565.536112:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319565.537651:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5197 00000100:00100000:0.0:1767319565.537661:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985201280:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319565.537684:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319565.546814:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5198 00000100:00100000:2.0:1767319565.546827:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985201536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319565.546852:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319565.547188:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985201024:12345-0@lo:38:kworker.0 Request processed in 11536us (12237us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319565.547198:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1789 00000100:00100000:0.0:1767319565.547425:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985201280:12345-0@lo:38:kworker.0 Request processed in 9762us (12005us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319565.547434:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5197 00000100:00080000:0.0:1767319565.547491:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319565.547496:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319565.547501:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985201024:0@lo:38:kworker.0 00000100:00080000:0.0:1767319565.547527:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319565.547530:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319565.547534:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985201280:0@lo:38:kworker.0 00000100:00100000:2.0:1767319565.547936:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985201536:12345-0@lo:38:kworker.0 Request processed in 1108us (12478us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319565.547944:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5198 00000100:00080000:0.0:1767319565.548676:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319565.548683:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319565.548686:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985201536:0@lo:38:kworker.0 00000100:00100000:0.0:1767319566.043745:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319566.043769:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985201920:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319566.043833:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319566.044012:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985201920 00000100:00100000:2.0:1767319566.044020:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985201920 00000100:00100000:2.0:1767319566.044086:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2254 00000100:00100000:2.0:1767319566.044095:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985201920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319566.044126:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319566.044191:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985201920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (357us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319566.044198:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2254 00000100:00100000:0.0:1767319566.044309:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985201920:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319566.044846:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985202048:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319566.045264:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319566.049437:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202048 00000100:00100000:2.0:1767319566.049453:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202048 00000100:00100000:2.0:1767319566.049584:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2255 00000100:00100000:2.0:1767319566.049595:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985202048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319566.049628:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319566.049831:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985202048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 237us (4938us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319566.049839:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2255 00000100:00100000:3.0:1767319566.050200:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985202048:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319566.175271:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319567.196958:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319568.220579:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319569.245715:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319569.578220:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319569.578262:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319569.578345:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568192 00000100:00100000:1.0:1767319569.578358:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568192 00000100:00100000:0.0:1767319569.578425:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568064 00000100:00100000:0.0:1767319569.578431:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568064 00000100:00080000:3.0:1767319569.578473:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319569.578525:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4697 00000100:00100000:1.0:1767319569.578559:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5199 00000100:00100000:0.0:1767319569.578561:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920568192:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319569.578570:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920568064:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319569.578592:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319569.578676:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920568192:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 140us (510us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319569.578684:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4697 00000100:00100000:1.0:1767319569.578704:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920568064:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 134us (438us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319569.578712:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5199 00010000:00010000:1.0:1767319570.271184:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319570.651394:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319570.651404:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319570.651408:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319570.651411:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319570.651416:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319570.651425:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4813 00000100:00080000:2.0:1767319570.651429:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319570.651437:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319570.651480:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ca80 x1853163985202176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319570.651507:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319570.651510:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319570.651512:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319570.651517:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4813 00000100:00080000:2.0:1767319570.651519:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319570.651523:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319570.651540:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18e300 x1853163985202304/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319570.651548:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985202176:0@lo:38:kworker.0 00000100:00080000:2.0:1767319570.651583:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319570.651588:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319570.651590:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319570.651595:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4813 00000100:00080000:2.0:1767319570.651598:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319570.651602:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319570.651615:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ea00 x1853163985202560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319570.651620:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319570.651635:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985202432:0@lo:400:kworker.0 00000100:00080000:2.0:1767319570.651653:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319570.651656:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319570.651658:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319570.651661:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4813 00000100:00080000:2.0:1767319570.651663:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319570.651667:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319570.651672:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319570.651681:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ce00 x1853163985202816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319570.651695:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985202304:0@lo:38:kworker.0 00000100:00100000:1.0:1767319570.651708:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985202944:0@lo:400:kworker.0 00000100:00100000:0.0:1767319570.651727:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319570.651732:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319570.651742:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202176 00000100:00100000:2.0:1767319570.651750:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202176 00000100:00100000:0.0:1767319570.651755:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985202560:0@lo:38:kworker.0 00000100:00100000:1.0:1767319570.651755:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319570.651760:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985202688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319570.651776:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319570.651787:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202304 00000100:00100000:1.0:1767319570.651791:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202304 00000100:00100000:1.0:1767319570.651821:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1790 00000100:00100000:1.0:1767319570.651827:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985202304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319570.651846:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319570.651854:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985202816:0@lo:38:kworker.0 00000100:00100000:0.0:1767319570.651873:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319570.651879:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985202304:12345-0@lo:38:kworker.0 Request processed in 52us (152us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319570.651887:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1790 00000100:00100000:1.0:1767319570.651911:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319570.651917:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985203072:0@lo:400:kworker.0 00000100:00100000:1.0:1767319570.651939:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319570.653216:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202432 00000100:00100000:2.0:1767319570.653225:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202432 00000100:00100000:2.0:1767319570.653762:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202560 00000100:00100000:2.0:1767319570.653766:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202560 00000100:00100000:2.0:1767319570.653790:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4698 00000100:00100000:2.0:1767319570.653801:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985202432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319570.653876:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985202432:12345-0@lo:400:kworker.0 Request processed in 78us (2206us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319570.653884:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4698 00000100:00100000:2.0:1767319570.653901:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5200 00000100:00100000:2.0:1767319570.653906:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985202176:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319570.653923:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319570.653948:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985202176:12345-0@lo:38:kworker.0 Request processed in 41us (2332us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319570.653956:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5200 00000100:00100000:2.0:1767319570.653969:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5201 00000100:00100000:2.0:1767319570.653973:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985202560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319570.653982:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319570.654005:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985202560:12345-0@lo:38:kworker.0 Request processed in 30us (2228us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319570.654011:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5201 00000100:00080000:2.0:1767319570.654061:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319570.654095:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202944 00000100:00100000:2.0:1767319570.654099:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202944 00000100:00100000:2.0:1767319570.654218:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203072 00000100:00100000:2.0:1767319570.654223:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203072 00000100:00100000:2.0:1767319570.654242:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3079 00000100:00100000:2.0:1767319570.654249:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985202944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319570.654427:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985202944:12345-0@lo:400:kworker.0 Request processed in 178us (2694us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319570.654433:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3079 00000100:00100000:2.0:1767319570.654446:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3080 00000100:00100000:2.0:1767319570.654452:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985203072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319570.654482:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985203072:12345-0@lo:400:kworker.0 Request processed in 30us (2541us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319570.654487:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3080 00000100:00080000:2.0:1767319570.654499:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319570.654501:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319570.654556:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985203072:0@lo:400:kworker.0 00000100:00100000:2.0:1767319570.654568:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985202432:0@lo:400:kworker.0 00000100:00080000:0.0:1767319570.655592:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319570.655602:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319570.655606:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985202176:0@lo:38:kworker.0 00000100:00080000:0.0:1767319570.655630:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319570.655634:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319570.655636:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985202304:0@lo:38:kworker.0 00000100:00080000:0.0:1767319570.655646:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319570.655649:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319570.655652:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985202560:0@lo:38:kworker.0 00000100:00100000:3.0:1767319570.655827:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202816 00000100:00100000:3.0:1767319570.655834:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202816 00000100:00100000:3.0:1767319570.655874:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5202 00000100:00100000:3.0:1767319570.655881:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985202816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319570.655898:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319570.655958:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985202816:12345-0@lo:38:kworker.0 Request processed in 76us (4083us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319570.655965:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5202 00000100:00080000:0.0:1767319570.656214:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319570.656219:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319570.656223:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985202816:0@lo:38:kworker.0 00000100:00100000:2.0:1767319570.657482:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319570.657548:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985202944:0@lo:400:kworker.0 00000100:00100000:1.0:1767319570.657604:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985202688 00000100:00100000:1.0:1767319570.657612:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985202688 00000100:00100000:1.0:1767319570.657726:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4699 00000100:00100000:1.0:1767319570.657737:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985202688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319570.657817:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985202688:12345-0@lo:400:kworker.0 Request processed in 82us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319570.657825:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4699 00000100:00100000:3.0:1767319570.657856:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985202688:0@lo:400:kworker.0 00000100:00080000:2.0:1767319570.661187:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319571.163519:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985203200:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319571.163539:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319571.163557:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985203328:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319571.163598:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319571.163650:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203200 00000100:00100000:2.0:1767319571.163656:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319571.163658:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203200 00000100:00100000:3.0:1767319571.163712:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203328 00000100:00100000:3.0:1767319571.163716:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203328 00000100:00100000:2.0:1767319571.163733:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2256 00000100:00100000:3.0:1767319571.163747:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2257 00000100:00100000:2.0:1767319571.163748:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985203200:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319571.163760:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985203328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319571.163781:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:3.0:1767319571.163782:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319571.163839:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985203200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (241us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319571.163847:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985203328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 89us (189us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319571.163848:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2256 00000100:00100000:3.0:1767319571.163873:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2257 00000100:00100000:2.0:1767319571.163896:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985203328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319571.163933:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985203200:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319571.291907:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319572.316713:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319573.340138:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319574.363537:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319574.701495:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319574.702092:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568320 00000100:00100000:3.0:1767319574.702105:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568320 00000100:00100000:3.0:1767319574.702352:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5203 00000100:00100000:3.0:1767319574.702363:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920568320:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319574.702392:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319574.702498:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920568320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 134us (1016us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319574.702507:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5203 00000100:00100000:1.0:1767319574.702950:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319574.705199:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568448 00000100:00100000:2.0:1767319574.705221:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568448 00000100:00100000:2.0:1767319574.705330:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4700 00000100:00100000:2.0:1767319574.705341:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920568448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319574.705434:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920568448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 94us (2483us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319574.705441:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4700 00000100:00080000:0.0:1767319574.705921:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319575.387447:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319575.771367:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319575.771380:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319575.771383:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319575.771387:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319575.771392:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319575.771402:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4818 00000100:00080000:2.0:1767319575.771408:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319575.771416:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319575.771478:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18fb80 x1853163985203456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319575.771502:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319575.771509:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319575.771514:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319575.771517:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4818 00000100:00080000:2.0:1767319575.771519:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319575.771522:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319575.771534:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18df80 x1853163985203584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319575.771578:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319575.771584:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319575.771586:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319575.771591:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4818 00000100:00080000:2.0:1767319575.771595:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319575.771599:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319575.771613:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ea00 x1853163985203840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319575.771642:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767319575.771644:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985203712:0@lo:400:kworker.0 00000100:00080000:2.0:1767319575.771646:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319575.771649:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319575.771659:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4818 00000100:00080000:2.0:1767319575.771668:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319575.771671:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319575.771679:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ca80 x1853163985204096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319575.771726:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985203968:0@lo:400:kworker.0 00000100:00100000:3.0:1767319575.771764:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319575.771798:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985204224:0@lo:400:kworker.0 00000100:00100000:2.0:1767319575.771855:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319575.771871:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319575.771884:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985204352:0@lo:400:kworker.0 00000100:00100000:0.0:1767319575.772032:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985203456:0@lo:38:kworker.0 00000100:00100000:1.0:1767319575.772154:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204224 00000100:00100000:3.0:1767319575.772215:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319575.772217:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204224 00000100:00100000:3.0:1767319575.772276:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203712 00000100:00100000:3.0:1767319575.772287:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203712 00000100:00100000:1.0:1767319575.772308:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204352 00000100:00100000:1.0:1767319575.772315:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204352 00000100:00100000:1.0:1767319575.772370:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3081 00000100:00100000:3.0:1767319575.772375:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4701 00000100:00100000:1.0:1767319575.772388:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985204224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319575.772395:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985203712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319575.772473:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985204224:12345-0@lo:400:kworker.0 Request processed in 89us (618us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319575.772483:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3081 00000100:00100000:1.0:1767319575.772502:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3082 00000100:00100000:1.0:1767319575.772520:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985204352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319575.772563:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985204352:12345-0@lo:400:kworker.0 Request processed in 55us (351us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319575.772569:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3082 00000100:00100000:1.0:1767319575.772636:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985204352:0@lo:400:kworker.0 00000100:00080000:1.0:1767319575.772677:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319575.772682:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319575.772685:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319575.772722:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319575.772769:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203968 00000100:00100000:1.0:1767319575.772773:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203968 00000100:00100000:1.0:1767319575.772798:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4702 00000100:00100000:1.0:1767319575.772803:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985203968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319575.772830:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985203968:12345-0@lo:400:kworker.0 Request processed in 27us (109us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319575.772835:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4702 00000100:00080000:1.0:1767319575.772848:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319575.772863:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985203968:0@lo:400:kworker.0 00000100:00100000:3.0:1767319575.773075:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985203712:12345-0@lo:400:kworker.0 Request processed in 684us (1308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319575.773085:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4701 00000100:00100000:1.0:1767319575.773727:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985203712:0@lo:400:kworker.0 00000100:00100000:2.0:1767319575.777815:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985204224:0@lo:400:kworker.0 00000100:00100000:1.0:1767319575.787867:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319575.788027:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985203584:0@lo:38:kworker.0 00000100:00100000:1.0:1767319575.788074:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319575.788220:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203584 00000100:00100000:2.0:1767319575.788239:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203584 00000100:00100000:2.0:1767319575.788303:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1791 00000100:00100000:2.0:1767319575.788315:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985203584:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319575.788346:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319575.788431:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985203584:12345-0@lo:38:kworker.0 Request processed in 116us (354us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319575.788439:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1791 00000100:00100000:3.0:1767319575.789386:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203456 00000100:00100000:3.0:1767319575.789398:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203456 00000100:00100000:3.0:1767319575.789443:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5204 00000100:00100000:3.0:1767319575.789450:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985203456:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319575.789474:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319575.789536:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985203456:12345-0@lo:38:kworker.0 Request processed in 86us (1678us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319575.789544:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5204 00000100:00080000:1.0:1767319575.792021:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319575.792038:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319575.792044:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985203584:0@lo:38:kworker.0 00000100:00100000:1.0:1767319575.792057:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985203840:0@lo:38:kworker.0 00000100:00100000:1.0:1767319575.792125:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319575.792230:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985204096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319575.792252:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319575.792357:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319575.792361:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319575.792364:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985203456:0@lo:38:kworker.0 00000100:00100000:3.0:1767319575.792453:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985203840 00000100:00100000:3.0:1767319575.792459:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985203840 00000100:00100000:3.0:1767319575.792499:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204096 00000100:00100000:1.0:1767319575.792676:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5205 00000100:00100000:1.0:1767319575.792683:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985203840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319575.792703:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319575.792764:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985203840:12345-0@lo:38:kworker.0 Request processed in 78us (637us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319575.792771:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5205 00000100:00100000:3.0:1767319575.793277:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204096 00000100:00100000:3.0:1767319575.793301:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5206 00000100:00100000:3.0:1767319575.793307:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985204096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319575.793319:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319575.793353:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985204096:12345-0@lo:38:kworker.0 Request processed in 46us (1100us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319575.793359:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5206 00000100:00080000:2.0:1767319575.794685:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319575.794704:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319575.794711:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985203840:0@lo:38:kworker.0 00000100:00080000:2.0:1767319575.794733:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319575.794737:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319575.794740:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985204096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319576.283888:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985204480:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319576.283974:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319576.284084:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319576.284105:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985204608:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319576.284236:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204480 00000100:00100000:0.0:1767319576.284248:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204480 00000100:00100000:2.0:1767319576.284256:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319576.285376:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204608 00000100:00100000:0.0:1767319576.285386:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204608 00000100:00100000:0.0:1767319576.285441:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2258 00000100:00100000:0.0:1767319576.285451:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985204480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319576.285484:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319576.285549:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985204480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (1573us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319576.285558:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2258 00000100:00100000:0.0:1767319576.285591:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2259 00000100:00100000:0.0:1767319576.285598:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985204608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319576.285615:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319576.285671:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985204608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (1419us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319576.285677:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2259 00000100:00100000:1.0:1767319576.285691:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985204480:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319576.285756:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985204608:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319576.413360:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319577.436322:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319578.459526:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319579.484879:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319579.822770:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319579.822848:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319579.825412:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568704 00000100:00100000:2.0:1767319579.825433:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568704 00000100:00100000:2.0:1767319579.825534:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568576 00000100:00100000:2.0:1767319579.825538:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568576 00000100:00100000:2.0:1767319579.825569:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4703 00000100:00100000:2.0:1767319579.825582:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920568704:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319579.825697:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920568704:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 117us (2941us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319579.825706:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4703 00000100:00100000:2.0:1767319579.825730:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5207 00000100:00100000:2.0:1767319579.825736:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920568576:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319579.825761:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319579.825794:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920568576:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (2946us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319579.825801:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5207 00000100:00080000:1.0:1767319579.827400:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319580.511035:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319580.891396:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319580.891410:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319580.891414:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319580.891419:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319580.891424:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319580.891437:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4824 00000100:00080000:2.0:1767319580.891442:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319580.891451:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319580.891522:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5597100 x1853163985204736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319580.891550:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319580.891554:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319580.891556:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319580.891560:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4824 00000100:00080000:2.0:1767319580.891562:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319580.891566:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319580.891580:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596680 x1853163985204864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319580.891610:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319580.891614:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319580.891616:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319580.891620:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4824 00000100:00080000:2.0:1767319580.891622:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319580.891626:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319580.891637:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594a80 x1853163985205120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319580.891667:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319580.891670:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319580.891672:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319580.891676:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4824 00000100:00080000:2.0:1767319580.891678:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319580.891682:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319580.891694:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319580.891694:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596d80 x1853163985205376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319580.891716:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985205248:0@lo:400:kworker.0 00000100:00100000:2.0:1767319580.891762:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319580.891772:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985205504:0@lo:400:kworker.0 00000100:00100000:2.0:1767319580.891825:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319580.891827:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319580.891977:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985205632:0@lo:400:kworker.0 00000100:00100000:0.0:1767319580.892009:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985205248 00000100:00100000:0.0:1767319580.892028:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985205248 00000100:00100000:0.0:1767319580.892109:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4704 00000100:00100000:0.0:1767319580.892120:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985205248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319580.898349:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985205504 00000100:00100000:1.0:1767319580.898374:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985205504 00000100:00100000:1.0:1767319580.898467:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3083 00000100:00100000:1.0:1767319580.898481:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985205504:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319580.898542:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985204992:0@lo:400:kworker.0 00000100:00100000:1.0:1767319580.898599:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985205504:12345-0@lo:400:kworker.0 Request processed in 121us (6777us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319580.898607:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319580.898614:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3083 00000100:00080000:1.0:1767319580.898646:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319580.898653:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319580.898683:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204992 00000100:00100000:2.0:1767319580.898687:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985204736:0@lo:38:kworker.0 00000100:00100000:1.0:1767319580.898688:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204992 00000100:00100000:1.0:1767319580.898734:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4705 00000100:00100000:2.0:1767319580.898735:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319580.898744:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985204992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319580.898763:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985204864:0@lo:38:kworker.0 00000100:00100000:1.0:1767319580.898792:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985204992:12345-0@lo:400:kworker.0 Request processed in 49us (189us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319580.898803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319580.898833:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985205120:0@lo:38:kworker.0 00000100:00100000:2.0:1767319580.898856:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319580.898873:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985205376:0@lo:38:kworker.0 00000100:00100000:2.0:1767319580.898895:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319580.898923:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204736 00000100:00100000:2.0:1767319580.898927:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204736 00000100:00100000:3.0:1767319580.899384:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319580.899808:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985204992:0@lo:400:kworker.0 00000100:00100000:1.0:1767319580.899973:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4705 00000100:00100000:1.0:1767319580.899994:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985205120 00000100:00100000:1.0:1767319580.899998:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985205120 00000100:00100000:3.0:1767319580.900022:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985205504:0@lo:400:kworker.0 00000100:00100000:3.0:1767319580.900056:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985205376 00000100:00100000:3.0:1767319580.900061:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985205376 00000100:00100000:0.0:1767319580.900113:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985205248:12345-0@lo:400:kworker.0 Request processed in 7993us (8287us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319580.900519:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5208 00000100:00100000:2.0:1767319580.900528:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985204736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319580.900553:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319580.900604:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985204736:12345-0@lo:38:kworker.0 Request processed in 75us (1872us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319580.900611:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5208 00000100:00080000:2.0:1767319580.900646:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319580.900653:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319580.900656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985204736:0@lo:38:kworker.0 00000100:00100000:2.0:1767319580.900705:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985204864 00000100:00100000:2.0:1767319580.900794:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985204864 00000100:00100000:2.0:1767319580.900823:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1792 00000100:00100000:2.0:1767319580.900829:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985204864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319580.900840:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319580.900875:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985204864:12345-0@lo:38:kworker.0 Request processed in 45us (2073us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319580.900881:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1792 00000100:00080000:2.0:1767319580.903047:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319580.903060:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319580.903097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985204864:0@lo:38:kworker.0 00000100:00100000:2.0:1767319580.903233:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985205632 00000100:00100000:2.0:1767319580.903242:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985205632 00000100:00100000:2.0:1767319580.903312:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3084 00000100:00100000:2.0:1767319580.903324:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985205632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319580.903431:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985205632:12345-0@lo:400:kworker.0 Request processed in 110us (4046us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319580.903439:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3084 00000100:00080000:2.0:1767319580.903464:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319580.903470:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319580.903506:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985205632:0@lo:400:kworker.0 00000100:00100000:1.0:1767319580.903574:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5209 00000100:00100000:1.0:1767319580.903585:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985205120:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319580.903593:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985205248:0@lo:400:kworker.0 00010000:00020000:1.0:1767319580.903608:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319580.903657:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985205120:12345-0@lo:38:kworker.0 Request processed in 73us (4801us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319580.903665:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5209 00000100:00100000:3.0:1767319580.904081:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5210 00000100:00100000:3.0:1767319580.904089:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985205376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319580.904103:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319580.904145:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985205376:12345-0@lo:38:kworker.0 Request processed in 55us (5249us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319580.904152:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5210 00000100:00100000:0.0:1767319580.904467:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4704 00000100:00080000:2.0:1767319580.905320:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319580.905329:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319580.905334:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985205120:0@lo:38:kworker.0 00000100:00080000:2.0:1767319580.905349:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319580.905431:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319580.905436:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985205376:0@lo:38:kworker.0 00000100:00100000:1.0:1767319581.403532:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319581.403602:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985205888:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319581.403641:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985205760:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319581.409106:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319581.409249:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985205888 00000100:00100000:2.0:1767319581.409263:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985205888 00000100:00100000:2.0:1767319581.409391:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2260 00000100:00100000:2.0:1767319581.409404:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985205888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319581.409449:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319581.409535:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985205888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 133us (439us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319581.409544:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2260 00000100:00100000:3.0:1767319581.409640:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985205888:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319581.413029:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319581.413205:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985205760 00000100:00100000:0.0:1767319581.413229:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985205760 00000100:00100000:0.0:1767319581.413301:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2261 00000100:00100000:0.0:1767319581.413314:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985205760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319581.413358:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319581.413601:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985205760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 290us (581us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319581.413611:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2261 00000100:00100000:1.0:1767319581.415723:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985205760:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319581.531472:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319582.557857:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319583.579387:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319584.603423:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319584.941794:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319584.941962:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319584.943069:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568960 00000100:00100000:2.0:1767319584.943085:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568960 00000100:00100000:2.0:1767319584.943225:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920568832 00000100:00100000:2.0:1767319584.943229:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920568832 00000100:00100000:2.0:1767319584.943258:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4706 00000100:00100000:2.0:1767319584.943269:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920568960:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319584.943380:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920568960:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 113us (1602us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319584.943609:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4706 00000100:00080000:1.0:1767319584.943660:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319584.943765:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5211 00000100:00100000:3.0:1767319584.943773:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920568832:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319584.943798:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319584.943830:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920568832:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 57us (1868us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319584.943837:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5211 00010000:00010000:1.0:1767319585.628029:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319586.013118:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319586.013134:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319586.013138:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319586.013145:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319586.013150:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319586.013183:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4829 00000100:00080000:3.0:1767319586.013190:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319586.013199:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319586.013261:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0e00 x1853163985206016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319586.013295:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319586.013299:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319586.013301:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319586.013305:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4829 00000100:00080000:3.0:1767319586.013308:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319586.013312:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319586.013327:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2680 x1853163985206144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319586.013378:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319586.013383:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319586.013385:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319586.013389:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4829 00000100:00080000:3.0:1767319586.013391:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319586.013395:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319586.013410:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593bb80 x1853163985206400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319586.013436:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319586.013440:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319586.013442:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319586.013447:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4829 00000100:00080000:3.0:1767319586.013450:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319586.013454:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319586.013458:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319586.013468:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593b800 x1853163985206656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319586.013477:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985206272:0@lo:400:kworker.0 00000100:00100000:3.0:1767319586.013542:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985206528:0@lo:400:kworker.0 00000100:00100000:1.0:1767319586.013580:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319586.013617:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319586.013659:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985206912:0@lo:400:kworker.0 00000100:00100000:1.0:1767319586.013686:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319586.013700:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985206784:0@lo:400:kworker.0 00000100:00100000:1.0:1767319586.013718:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206272 00000100:00100000:1.0:1767319586.013725:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206272 00000100:00100000:2.0:1767319586.013742:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319586.013791:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206528 00000100:00100000:1.0:1767319586.013793:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206528 00000100:00100000:3.0:1767319586.013850:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206912 00000100:00100000:3.0:1767319586.013856:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206912 00000100:00100000:3.0:1767319586.013892:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206784 00000100:00100000:3.0:1767319586.013896:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206784 00000100:00100000:3.0:1767319586.013925:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3085 00000100:00100000:3.0:1767319586.013935:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985206912:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319586.014017:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985206912:12345-0@lo:400:kworker.0 Request processed in 83us (329us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319586.014026:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3085 00000100:00100000:3.0:1767319586.014045:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3086 00000100:00100000:3.0:1767319586.014052:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985206784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319586.014084:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985206784:12345-0@lo:400:kworker.0 Request processed in 33us (343us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319586.014090:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3086 00000100:00100000:3.0:1767319586.014137:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985206912:0@lo:400:kworker.0 00000100:00100000:3.0:1767319586.014215:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985206784:0@lo:400:kworker.0 00000100:00080000:3.0:1767319586.014251:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319586.014338:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319586.014341:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319586.022815:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4707 00000100:00100000:1.0:1767319586.022848:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985206272:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319586.022992:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985206272:12345-0@lo:400:kworker.0 Request processed in 159us (9412us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319586.023001:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4707 00000100:00100000:1.0:1767319586.023027:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4708 00000100:00100000:1.0:1767319586.023035:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985206528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319586.023083:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985206528:12345-0@lo:400:kworker.0 Request processed in 49us (9467us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319586.023090:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4708 00000100:00100000:1.0:1767319586.023152:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985206016:0@lo:38:kworker.0 00000100:00100000:1.0:1767319586.023222:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319586.023266:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985206144:0@lo:38:kworker.0 00000100:00100000:1.0:1767319586.023297:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319586.023333:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985206400:0@lo:38:kworker.0 00000100:00100000:1.0:1767319586.023394:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319586.023417:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985206656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319586.023440:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319586.023477:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206016 00000100:00100000:1.0:1767319586.023487:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206016 00000100:00100000:1.0:1767319586.023509:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206400 00000100:00100000:1.0:1767319586.023512:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206400 00000100:00100000:1.0:1767319586.023519:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206656 00000100:00100000:1.0:1767319586.023521:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206656 00000100:00100000:1.0:1767319586.023532:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5212 00000100:00100000:1.0:1767319586.023536:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985206016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319586.023560:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319586.023592:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985206016:12345-0@lo:38:kworker.0 Request processed in 54us (370us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319586.023598:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5212 00000100:00080000:1.0:1767319586.023632:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319586.023637:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319586.023641:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985206016:0@lo:38:kworker.0 00000100:00100000:1.0:1767319586.023670:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5213 00000100:00100000:1.0:1767319586.023675:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985206400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319586.023684:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319586.023710:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985206400:12345-0@lo:38:kworker.0 Request processed in 33us (315us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319586.023716:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5213 00000100:00080000:1.0:1767319586.023734:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319586.023738:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319586.023741:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985206400:0@lo:38:kworker.0 00000100:00100000:1.0:1767319586.023759:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5214 00000100:00100000:1.0:1767319586.023764:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985206656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319586.023772:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319586.023799:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985206656:12345-0@lo:38:kworker.0 Request processed in 34us (357us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319586.023805:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5214 00000100:00080000:1.0:1767319586.023824:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319586.023828:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319586.023831:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985206656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319586.023871:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985206144 00000100:00100000:1.0:1767319586.023874:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985206144 00000100:00100000:1.0:1767319586.023898:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1793 00000100:00100000:1.0:1767319586.023902:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985206144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319586.023912:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319586.023945:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985206144:12345-0@lo:38:kworker.0 Request processed in 41us (647us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319586.023955:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1793 00000100:00080000:1.0:1767319586.023991:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319586.023996:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319586.024000:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985206144:0@lo:38:kworker.0 00000100:00100000:3.0:1767319586.024022:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985206528:0@lo:400:kworker.0 00000100:00100000:3.0:1767319586.024066:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985206272:0@lo:400:kworker.0 00000100:00100000:1.0:1767319586.524507:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319586.524541:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985207040:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319586.524639:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319586.524750:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985207040 00000100:00100000:3.0:1767319586.524761:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985207040 00000100:00100000:3.0:1767319586.524836:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2262 00000100:00100000:3.0:1767319586.524849:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985207040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319586.524889:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319586.524972:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985207040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 125us (332us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319586.524980:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2262 00000100:00100000:2.0:1767319586.525080:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985207040:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319586.525720:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319586.525733:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985207168:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319586.525774:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319586.525900:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985207168 00000100:00100000:1.0:1767319586.525907:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985207168 00000100:00100000:1.0:1767319586.525951:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2263 00000100:00100000:1.0:1767319586.525967:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985207168:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319586.525992:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319586.526077:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985207168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 119us (302us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319586.526085:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2263 00000100:00100000:2.0:1767319586.526609:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985207168:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319586.651763:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319587.678218:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319588.699953:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319589.724674:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319590.057865:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319590.057995:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920569216 00000100:00100000:0.0:1767319590.058013:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920569216 00000100:00100000:0.0:1767319590.058312:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4709 00000100:00100000:0.0:1767319590.058327:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920569216:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319590.058446:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920569216:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 122us (591us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319590.058468:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4709 00000100:00080000:3.0:1767319590.064966:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319590.065294:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767319590.065340:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920569344 offset 0 length 520: 4 00000100:00100000:0.0:1767319590.065437:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920569088 00000100:00100000:0.0:1767319590.065450:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920569088 00000100:00100000:0.0:1767319590.065546:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5215 00000100:00100000:0.0:1767319590.065556:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920569088:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319590.065586:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319590.065670:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920569088:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 114us (387us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319590.065679:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5215 00000100:00100000:1.0:1767319590.107357:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319565/real 1767319565] req@ffff8ce1a612d180 x1853163985200768/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319590 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319590.107404:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a612d180 x1853163985200768/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319590 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319590.107432:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319590.107456:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319590.107461:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985200768:0@lo:250:kworker.0 00000100:00080000:0.0:1767319590.107620:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319590.107627:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319590.107631:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319590.107636:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4813 00000100:00080000:0.0:1767319590.107641:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319590.107649:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319590.107695:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b100 x1853163985207296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319590.107750:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985207296:0@lo:250:kworker.0 00000400:00000100:1.0:1767319590.107808:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985207296 offset 0 length 520: 4 00000100:00080000:0.0:1767319590.107877:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319590.107881:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319590.107883:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319590.107887:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4834 00000100:00080000:0.0:1767319590.107913:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319590.107918:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319590.107938:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a680 x1853163985207424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319590.108083:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985207424:0@lo:38:kworker.0 00000100:00100000:1.0:1767319590.108123:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319590.108243:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985207424 00000100:00100000:2.0:1767319590.108254:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985207424 00000100:00100000:2.0:1767319590.108304:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5216 00000100:00100000:2.0:1767319590.108313:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985207424:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319590.108338:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319590.108387:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985207424:12345-0@lo:38:kworker.0 Request processed in 73us (264us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319590.108396:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5216 00000100:00080000:1.0:1767319590.108507:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319590.108515:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319590.108519:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985207424:0@lo:38:kworker.0 00000100:00080000:0.0:1767319590.108677:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319590.108682:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319590.108684:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319590.108688:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4834 00000100:00080000:0.0:1767319590.108690:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319590.108694:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319590.108711:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88380 x1853163985207552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319590.108753:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985207552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319590.108784:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319590.108843:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985207552 00000100:00100000:2.0:1767319590.108847:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985207552 00000100:00100000:2.0:1767319590.108879:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1794 00000100:00100000:2.0:1767319590.108884:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985207552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319590.108922:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319590.109023:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319590.109027:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319590.109029:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319590.109033:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4834 00000100:00080000:0.0:1767319590.109036:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319590.109040:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319590.109053:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a300 x1853163985207808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319590.109091:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985207808:0@lo:38:kworker.0 00000100:00100000:1.0:1767319590.109117:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319590.109201:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985207552:12345-0@lo:38:kworker.0 Request processed in 315us (415us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319590.109209:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1794 00000100:00100000:2.0:1767319590.109246:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319590.109254:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985207680:0@lo:400:kworker.0 00000100:00100000:2.0:1767319590.109281:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319590.109309:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985207808 00000100:00100000:2.0:1767319590.109314:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985207808 00000100:00100000:2.0:1767319590.109339:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985207680 00000100:00100000:2.0:1767319590.109344:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985207680 00000100:00100000:3.0:1767319590.109371:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5217 00000100:00100000:3.0:1767319590.109385:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985207808:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319590.109398:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4710 00000100:00100000:2.0:1767319590.109407:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985207680:12345-0@lo:400:kworker.0 00010000:00020000:3.0:1767319590.109412:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319590.109476:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985207680:12345-0@lo:400:kworker.0 Request processed in 67us (191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319590.109485:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4710 00000100:00100000:3.0:1767319590.109517:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985207808:12345-0@lo:38:kworker.0 Request processed in 130us (395us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319590.109526:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5217 00000100:00100000:2.0:1767319590.109536:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985207680:0@lo:400:kworker.0 00000100:00080000:3.0:1767319590.109561:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319590.109626:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319590.109632:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319590.109638:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985207552:0@lo:38:kworker.0 00000100:00080000:1.0:1767319590.109660:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319590.109664:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319590.109666:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985207808:0@lo:38:kworker.0 00000100:00100000:2.0:1767319590.110032:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985207936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319590.110066:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319590.110121:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985207936 00000100:00100000:1.0:1767319590.110126:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985207936 00000100:00100000:1.0:1767319590.110188:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4711 00000100:00100000:1.0:1767319590.110196:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985207936:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319590.110272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319590.110331:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319590.110336:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319590.110339:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319590.110343:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4834 00000100:00080000:0.0:1767319590.110345:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319590.110349:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319590.110367:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a300 x1853163985208064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319590.110422:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985208192:0@lo:400:kworker.0 00000100:00100000:3.0:1767319590.110456:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319590.110491:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208192 00000100:00100000:3.0:1767319590.110495:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208192 00000100:00100000:3.0:1767319590.110543:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3087 00000100:00100000:3.0:1767319590.110550:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985208192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319590.110585:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985208192:12345-0@lo:400:kworker.0 Request processed in 35us (128us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319590.110591:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3087 00000100:00080000:3.0:1767319590.110608:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319590.110630:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985208192:0@lo:400:kworker.0 00000100:00100000:3.0:1767319590.110658:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985208064:0@lo:38:kworker.0 00000100:00100000:3.0:1767319590.110683:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319590.110707:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208064 00000100:00100000:3.0:1767319590.110710:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208064 00000100:00100000:3.0:1767319590.110730:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5218 00000100:00100000:3.0:1767319590.110734:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985208064:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319590.110751:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319590.110779:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985208064:12345-0@lo:38:kworker.0 Request processed in 44us (96us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319590.110785:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5218 00000100:00080000:3.0:1767319590.110806:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319590.110809:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319590.110813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985208064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319590.110917:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985207936:12345-0@lo:400:kworker.0 Request processed in 721us (849us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319590.110924:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4711 00000100:00100000:3.0:1767319590.111187:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985208320:0@lo:400:kworker.0 00000100:00100000:3.0:1767319590.111216:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319590.111331:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208320 00000100:00100000:0.0:1767319590.111336:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208320 00000100:00100000:0.0:1767319590.111380:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3088 00000100:00100000:0.0:1767319590.111387:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a04450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985208320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319590.111427:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a04450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985208320:12345-0@lo:400:kworker.0 Request processed in 40us (209us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319590.111433:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3088 00000100:00100000:0.0:1767319590.111675:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985208320:0@lo:400:kworker.0 00000100:00080000:3.0:1767319590.111735:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319590.112435:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985207936:0@lo:400:kworker.0 00010000:00010000:2.0:1767319590.748699:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319591.643696:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985208448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319591.643792:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319591.643979:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319591.643996:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985208576:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319591.644050:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319591.646253:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208448 00000100:00100000:3.0:1767319591.646273:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208448 00000100:00100000:3.0:1767319591.646343:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208576 00000100:00100000:3.0:1767319591.646346:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208576 00000100:00100000:3.0:1767319591.646433:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2264 00000100:00100000:3.0:1767319591.646445:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985208448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319591.646486:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319591.646565:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985208448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 123us (2773us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319591.646573:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2264 00000100:00100000:3.0:1767319591.646591:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2265 00000100:00100000:3.0:1767319591.646596:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985208576:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319591.646612:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319591.646639:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985208576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (2589us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319591.646645:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2265 00000100:00100000:0.0:1767319591.646745:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985208448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319591.646790:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985208576:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319591.771377:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319592.796419:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319593.819571:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319594.843553:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319595.182916:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319595.182974:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319595.183079:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920569600 00000100:00100000:3.0:1767319595.183091:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920569600 00000100:00100000:3.0:1767319595.183213:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920569472 00000100:00100000:3.0:1767319595.183216:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920569472 00000100:00100000:3.0:1767319595.183237:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4712 00000100:00100000:3.0:1767319595.183248:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920569600:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319595.183338:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920569600:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (432us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319595.183345:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4712 00000100:00100000:3.0:1767319595.183364:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5219 00000100:00100000:3.0:1767319595.183368:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920569472:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319595.183389:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319595.183416:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920569472:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (441us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319595.183422:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5219 00000100:00080000:0.0:1767319595.183896:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319595.227336:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319595.227347:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319595.227350:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319595.227354:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319595.227358:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319595.227366:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4838 00000100:00080000:0.0:1767319595.227370:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319595.227377:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319595.227414:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88a80 x1853163985208704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319595.227449:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319595.227453:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319595.227454:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319595.227457:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4838 00000100:00080000:0.0:1767319595.227460:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319595.227463:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319595.227486:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985208704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319595.227490:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985208832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319595.227536:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319595.227539:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319595.227541:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319595.227545:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4838 00000100:00080000:0.0:1767319595.227548:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319595.227551:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319595.227557:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319595.227564:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89880 x1853163985209088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319595.227594:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319595.227597:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319595.227599:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319595.227602:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4838 00000100:00080000:0.0:1767319595.227605:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319595.227608:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319595.227620:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985208832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319595.227621:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b480 x1853163985209344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319595.227657:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319595.227678:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985209472:0@lo:400:kworker.0 00000100:00100000:3.0:1767319595.227698:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985209088:0@lo:38:kworker.0 00000100:00100000:0.0:1767319595.227714:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319595.227738:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319595.227745:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985209216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319595.227752:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319595.227775:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319595.227847:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208832 00000100:00100000:0.0:1767319595.227854:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208832 00000100:00100000:1.0:1767319595.228211:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985209600:0@lo:400:kworker.0 00000100:00100000:1.0:1767319595.228242:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319595.228266:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319595.228281:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985208960:0@lo:400:kworker.0 00000100:00100000:1.0:1767319595.228304:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319595.228332:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208704 00000100:00100000:1.0:1767319595.228336:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208704 00000100:00100000:1.0:1767319595.228356:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209088 00000100:00100000:1.0:1767319595.228359:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209088 00000100:00100000:1.0:1767319595.228371:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209216 00000100:00100000:1.0:1767319595.228373:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209216 00000100:00100000:1.0:1767319595.228400:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985208960 00000100:00100000:1.0:1767319595.228403:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985208960 00000100:00100000:1.0:1767319595.228417:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4713 00000100:00100000:1.0:1767319595.228422:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985209216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319595.228467:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985209344:0@lo:38:kworker.0 00000100:00100000:1.0:1767319595.228475:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985209216:12345-0@lo:400:kworker.0 Request processed in 51us (699us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319595.228483:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4713 00000100:00100000:1.0:1767319595.228505:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4714 00000100:00100000:3.0:1767319595.228506:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319595.228517:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985208960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319595.228557:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985208960:12345-0@lo:400:kworker.0 Request processed in 44us (252us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319595.228559:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209344 00000100:00100000:1.0:1767319595.228565:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4714 00000100:00100000:3.0:1767319595.228566:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209344 00000100:00100000:1.0:1767319595.228582:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5220 00000100:00100000:1.0:1767319595.228588:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985208704:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319595.228593:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5221 00000100:00100000:3.0:1767319595.228601:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985209088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319595.228610:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767319595.228614:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319595.228859:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985208704:12345-0@lo:38:kworker.0 Request processed in 269us (1303us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319595.228867:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5220 00000100:00100000:1.0:1767319595.228880:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5222 00000100:00100000:1.0:1767319595.228884:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985209344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319595.228977:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319595.229114:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985209088:12345-0@lo:38:kworker.0 Request processed in 510us (1360us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319595.229322:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5221 00000100:00100000:3.0:1767319595.229350:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209472 00000100:00100000:3.0:1767319595.229355:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209472 00000100:00100000:3.0:1767319595.229383:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209600 00000100:00100000:3.0:1767319595.229386:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209600 00000100:00100000:3.0:1767319595.229405:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3089 00000100:00100000:3.0:1767319595.229412:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985209472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319595.229446:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985209472:12345-0@lo:400:kworker.0 Request processed in 34us (1732us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319595.229452:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3089 00000100:00100000:3.0:1767319595.229464:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3090 00000100:00100000:3.0:1767319595.229470:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985209600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319595.229496:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985209600:12345-0@lo:400:kworker.0 Request processed in 26us (1254us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319595.229501:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3090 00000100:00100000:0.0:1767319595.229561:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1795 00000100:00100000:0.0:1767319595.229571:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985208832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319595.229582:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319595.229594:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985208960:0@lo:400:kworker.0 00000100:00100000:3.0:1767319595.229609:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985209600:0@lo:400:kworker.0 00000100:00100000:0.0:1767319595.229621:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985208832:12345-0@lo:38:kworker.0 Request processed in 50us (1963us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767319595.229636:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319595.229641:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319595.229644:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319595.229662:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985209216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319595.229671:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985209472:0@lo:400:kworker.0 00000100:00100000:1.0:1767319595.233625:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985209344:12345-0@lo:38:kworker.0 Request processed in 4728us (5114us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319595.233638:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5222 00000100:00080000:1.0:1767319595.233714:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319595.233720:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319595.233724:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985209344:0@lo:38:kworker.0 00000100:00080000:1.0:1767319595.233735:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319595.233739:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319595.233741:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985208832:0@lo:38:kworker.0 00000100:00080000:1.0:1767319595.233749:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319595.233752:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319595.233755:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985209088:0@lo:38:kworker.0 00000100:00080000:1.0:1767319595.233763:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319595.233766:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319595.233769:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985208704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319595.239605:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1795 00010000:00010000:1.0:1767319595.867871:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319596.764403:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319596.764435:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985209856:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319596.764505:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319596.764792:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985209728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319596.764822:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319596.765074:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209856 00000100:00100000:0.0:1767319596.765082:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209856 00000100:00100000:0.0:1767319596.765128:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209728 00000100:00100000:0.0:1767319596.765131:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209728 00000100:00100000:0.0:1767319596.765181:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2266 00000100:00100000:0.0:1767319596.765191:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985209856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319596.765220:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319596.765266:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985209856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (760us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319596.765274:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2266 00000100:00100000:0.0:1767319596.765290:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2267 00000100:00100000:0.0:1767319596.765296:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985209728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319596.765313:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319596.765343:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985209728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (520us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319596.765349:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2267 00000100:00100000:0.0:1767319596.765388:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985209728:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319596.765441:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985209856:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319596.908942:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319597.915395:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319598.939441:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319599.963808:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319600.298401:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319600.298451:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319600.298605:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920569728 00000100:00100000:2.0:1767319600.298616:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920569728 00000100:00100000:2.0:1767319600.298663:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920569856 00000100:00100000:2.0:1767319600.298667:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920569856 00000100:00100000:2.0:1767319600.298724:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4715 00000100:00100000:2.0:1767319600.298735:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920569856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319600.298817:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920569856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (364us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319600.298825:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4715 00000100:00100000:2.0:1767319600.298844:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5223 00000100:00100000:2.0:1767319600.298849:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920569728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319600.298869:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319600.298896:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920569728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (508us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319600.298903:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5223 00000100:00080000:0.0:1767319600.299105:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319600.347534:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319600.347544:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319600.347547:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319600.347551:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319600.347556:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319600.347564:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4843 00000100:00080000:2.0:1767319600.347569:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319600.347576:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319600.347616:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594000 x1853163985209984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319600.347643:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319600.347646:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319600.347648:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319600.347651:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4843 00000100:00080000:2.0:1767319600.347653:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319600.347657:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319600.347671:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594e00 x1853163985210112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319600.347739:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985210240:0@lo:400:kworker.0 00000100:00100000:0.0:1767319600.347810:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319600.347916:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985210240 00000100:00100000:3.0:1767319600.347925:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985210240 00000100:00100000:3.0:1767319600.347995:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4716 00000100:00100000:3.0:1767319600.348004:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985210240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.348086:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985209984:0@lo:38:kworker.0 00000100:00100000:1.0:1767319600.348116:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.348137:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985210112:0@lo:38:kworker.0 00000100:00100000:1.0:1767319600.348161:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.348214:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985209984 00000100:00100000:1.0:1767319600.348218:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985209984 00000100:00100000:1.0:1767319600.348241:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5224 00000100:00100000:1.0:1767319600.348246:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985209984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319600.348265:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319600.348301:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985209984:12345-0@lo:38:kworker.0 Request processed in 54us (184us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319600.348308:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5224 00000100:00080000:1.0:1767319600.348342:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319600.348347:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319600.348351:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985209984:0@lo:38:kworker.0 00000100:00080000:1.0:1767319600.348389:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319600.348405:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985210112 00000100:00100000:1.0:1767319600.348409:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985210112 00000100:00100000:1.0:1767319600.348431:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1796 00000100:00100000:1.0:1767319600.348436:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985210112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319600.348447:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319600.348475:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985210112:12345-0@lo:38:kworker.0 Request processed in 38us (314us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319600.348481:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1796 00000100:00080000:1.0:1767319600.348503:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319600.348507:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319600.348510:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985210112:0@lo:38:kworker.0 00000100:00080000:2.0:1767319600.348763:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319600.348768:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319600.348770:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319600.348775:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4843 00000100:00080000:2.0:1767319600.348777:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319600.348782:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319600.348801:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596a00 x1853163985210368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319600.348831:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319600.348835:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319600.348837:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319600.348840:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4843 00000100:00080000:2.0:1767319600.348843:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319600.348846:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319600.348858:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594e00 x1853163985210624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319600.349062:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985210880:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349097:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.349133:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985210496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349157:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.349193:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319600.349198:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985210752:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349219:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.349245:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985210880 00000100:00100000:1.0:1767319600.349249:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985210880 00000100:00100000:1.0:1767319600.349285:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985210752 00000100:00100000:1.0:1767319600.349289:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985210752 00000100:00100000:1.0:1767319600.349310:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3091 00000100:00100000:1.0:1767319600.349316:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985210880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349350:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985210880:12345-0@lo:400:kworker.0 Request processed in 34us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319600.349356:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3091 00000100:00100000:1.0:1767319600.349366:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3092 00000100:00100000:1.0:1767319600.349370:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985210752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349402:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985210752:12345-0@lo:400:kworker.0 Request processed in 31us (182us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319600.349408:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3092 00000100:00100000:1.0:1767319600.349431:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985210496 00000100:00100000:1.0:1767319600.349435:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985210496 00000100:00100000:1.0:1767319600.349460:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4717 00000100:00100000:1.0:1767319600.349466:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985210496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349494:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985210496:12345-0@lo:400:kworker.0 Request processed in 28us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319600.349500:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4717 00000100:00080000:1.0:1767319600.349514:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319600.349517:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319600.349519:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319600.349536:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985210880:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349555:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985210752:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349562:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985210496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.349578:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985210368:0@lo:38:kworker.0 00000100:00100000:1.0:1767319600.349599:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.349614:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985210624:0@lo:38:kworker.0 00000100:00100000:1.0:1767319600.349633:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.349657:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985210368 00000100:00100000:1.0:1767319600.349661:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985210368 00000100:00100000:1.0:1767319600.349677:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985210624 00000100:00100000:1.0:1767319600.349680:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985210624 00000100:00100000:1.0:1767319600.349697:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5225 00000100:00100000:1.0:1767319600.349701:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985210368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319600.349729:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319600.349761:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985210368:12345-0@lo:38:kworker.0 Request processed in 59us (161us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319600.349768:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5225 00000100:00100000:1.0:1767319600.349780:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5226 00000100:00100000:1.0:1767319600.349784:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985210624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319600.349793:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319600.349817:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985210624:12345-0@lo:38:kworker.0 Request processed in 31us (183us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319600.349823:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5226 00000100:00080000:1.0:1767319600.349853:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319600.349858:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319600.349861:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985210368:0@lo:38:kworker.0 00000100:00080000:1.0:1767319600.349868:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319600.349871:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319600.349874:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985210624:0@lo:38:kworker.0 00000100:00100000:3.0:1767319600.350248:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985210240:12345-0@lo:400:kworker.0 Request processed in 2244us (2436us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319600.350259:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4716 00000100:00100000:0.0:1767319600.356209:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985210240:0@lo:400:kworker.0 00010000:00010000:1.0:1767319600.990158:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319601.883495:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985211008:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319601.883571:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319601.883586:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319601.883604:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985211136:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319601.883635:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211008 00000100:00100000:3.0:1767319601.883642:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319601.883645:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211008 00000100:00100000:1.0:1767319601.883779:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211136 00000100:00100000:1.0:1767319601.883783:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211136 00000100:00100000:3.0:1767319601.884814:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2268 00000100:00100000:3.0:1767319601.884833:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985211008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319601.884959:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319601.885097:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985211008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 271us (1525us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319601.885106:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2268 00000100:00100000:3.0:1767319601.885127:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2269 00000100:00100000:3.0:1767319601.885134:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985211136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319601.885150:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319601.885202:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985211136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (1562us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319601.885208:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2269 00000100:00100000:3.0:1767319601.885253:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985211136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319601.888350:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985211008:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319602.011404:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319603.037839:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319604.061020:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319605.086852:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319605.417695:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319605.417826:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920570112 00000100:00100000:1.0:1767319605.417837:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920570112 00000100:00100000:1.0:1767319605.417930:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4718 00000100:00100000:1.0:1767319605.417941:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920570112:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319605.418016:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920570112:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319605.418023:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4718 00000100:00080000:3.0:1767319605.418345:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319605.418522:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319605.418575:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920569984 00000100:00100000:0.0:1767319605.418582:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920569984 00000100:00100000:0.0:1767319605.418628:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5227 00000100:00100000:0.0:1767319605.418635:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920569984:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319605.418656:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319605.418710:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920569984:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 75us (188us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319605.418717:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5227 00000100:00080000:0.0:1767319605.468096:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319605.468107:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319605.468114:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319605.468120:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319605.468125:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319605.468136:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4848 00000100:00080000:0.0:1767319605.468140:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319605.468147:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319605.468232:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89880 x1853163985211264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319605.468249:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319605.468252:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319605.468254:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319605.468257:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4848 00000100:00080000:0.0:1767319605.468260:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319605.468263:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319605.468272:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985211392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319605.468306:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319605.468310:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319605.468311:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319605.468315:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4848 00000100:00080000:0.0:1767319605.468317:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319605.468320:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319605.468333:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89180 x1853163985211648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319605.468355:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319605.468359:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319605.468361:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319605.468365:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4848 00000100:00080000:0.0:1767319605.468367:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319605.468371:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319605.468379:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b800 x1853163985211904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319605.468443:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985212160:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.468512:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.468566:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319605.468571:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985211520:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.468593:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.468650:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985212032:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.468671:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.468686:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319605.468691:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985211776:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.468706:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.468730:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212160 00000100:00100000:0.0:1767319605.468736:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212160 00000100:00100000:0.0:1767319605.468778:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212032 00000100:00100000:0.0:1767319605.468780:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212032 00000100:00100000:0.0:1767319605.468798:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3093 00000100:00100000:0.0:1767319605.468806:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985212160:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.468852:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985212160:12345-0@lo:400:kworker.0 Request processed in 46us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319605.468871:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3093 00000100:00100000:0.0:1767319605.468889:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3094 00000100:00100000:0.0:1767319605.468896:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985212032:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.468927:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985212032:12345-0@lo:400:kworker.0 Request processed in 32us (255us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319605.468932:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3094 00000100:00100000:0.0:1767319605.469076:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985212032:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.469103:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211520 00000100:00100000:0.0:1767319605.469106:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211520 00000100:00100000:0.0:1767319605.469133:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211776 00000100:00100000:0.0:1767319605.469136:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211776 00000100:00100000:0.0:1767319605.469158:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4719 00000100:00100000:0.0:1767319605.469187:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985211520:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319605.469198:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319605.469205:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319605.469208:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319605.469225:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985211520:12345-0@lo:400:kworker.0 Request processed in 61us (630us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319605.469229:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4720 00000100:00100000:0.0:1767319605.469238:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4719 00000100:00100000:3.0:1767319605.469245:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985211776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.469271:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985211520:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.469284:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985212160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319605.469296:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985211776:12345-0@lo:400:kworker.0 Request processed in 56us (586us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319605.469304:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4720 00000100:00100000:0.0:1767319605.469333:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985211776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319605.470146:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985211264:0@lo:38:kworker.0 00000100:00100000:2.0:1767319605.470207:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319605.470255:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985211392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319605.470279:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319605.470316:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985211648:0@lo:38:kworker.0 00000100:00100000:2.0:1767319605.470339:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319605.470366:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985211904:0@lo:38:kworker.0 00000100:00100000:2.0:1767319605.470387:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319605.470424:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211264 00000100:00100000:2.0:1767319605.470429:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211264 00000100:00100000:2.0:1767319605.470451:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211648 00000100:00100000:2.0:1767319605.470454:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211648 00000100:00100000:2.0:1767319605.470466:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211904 00000100:00100000:2.0:1767319605.470468:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211904 00000100:00100000:2.0:1767319605.470481:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5228 00000100:00100000:2.0:1767319605.470487:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985211264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319605.470506:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319605.470541:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985211264:12345-0@lo:38:kworker.0 Request processed in 52us (333us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319605.470548:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5228 00000100:00100000:2.0:1767319605.470565:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5229 00000100:00100000:2.0:1767319605.470568:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985211648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319605.470577:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319605.470600:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985211648:12345-0@lo:38:kworker.0 Request processed in 31us (261us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319605.470607:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5229 00000100:00100000:2.0:1767319605.470632:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5230 00000100:00100000:2.0:1767319605.470636:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985211904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319605.470645:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319605.470672:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985211904:12345-0@lo:38:kworker.0 Request processed in 34us (284us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319605.470678:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5230 00000100:00080000:2.0:1767319605.470708:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319605.470713:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319605.470716:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985211264:0@lo:38:kworker.0 00000100:00080000:2.0:1767319605.470725:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319605.470728:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319605.470730:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985211648:0@lo:38:kworker.0 00000100:00080000:2.0:1767319605.470737:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319605.470740:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319605.470742:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985211904:0@lo:38:kworker.0 00000100:00100000:3.0:1767319605.472085:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985211392 00000100:00100000:3.0:1767319605.472096:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985211392 00000100:00100000:3.0:1767319605.472150:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1797 00000100:00100000:3.0:1767319605.472157:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985211392:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319605.472196:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319605.472264:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985211392:12345-0@lo:38:kworker.0 Request processed in 105us (1981us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319605.472270:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1797 00000100:00080000:2.0:1767319605.472512:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319605.472518:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319605.472521:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985211392:0@lo:38:kworker.0 00010000:00010000:0.0:1767319606.107509:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319607.003526:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319607.003552:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985212288:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319607.003656:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319607.004865:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319607.004882:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985212416:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319607.004914:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212288 00000100:00100000:3.0:1767319607.004925:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212288 00000100:00100000:2.0:1767319607.004936:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319607.004982:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212416 00000100:00100000:3.0:1767319607.004986:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212416 00000100:00080000:2.0:1767319607.004994:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319607.005000:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319607.005009:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2270 00000100:00100000:3.0:1767319607.005019:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985212288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319607.005047:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319607.005066:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2271 00000100:00100000:0.0:1767319607.005079:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985212416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319607.005100:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319607.005109:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985212288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (1451us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319607.005118:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2270 00000100:00100000:0.0:1767319607.005146:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985212416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (210us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319607.005156:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2271 00000100:00100000:2.0:1767319607.005340:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985212288:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319607.007417:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985212416:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319607.131388:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319608.155396:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319609.179474:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319610.203476:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319610.537791:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319610.538197:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319610.539130:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920570368 00000100:00100000:2.0:1767319610.539146:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920570368 00000100:00100000:2.0:1767319610.539345:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920570240 00000100:00100000:2.0:1767319610.539350:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920570240 00000100:00100000:2.0:1767319610.539372:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4721 00000100:00100000:2.0:1767319610.539383:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920570368:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319610.539470:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920570368:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (1689us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319610.539478:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4721 00000100:00100000:2.0:1767319610.539507:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5231 00000100:00100000:2.0:1767319610.539512:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920570240:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319610.539539:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319610.539566:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920570240:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (1401us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319610.539572:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5231 00000100:00080000:0.0:1767319610.540460:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319610.587446:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319610.587456:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319610.587459:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319610.587463:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319610.587467:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319610.587474:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4853 00000100:00080000:0.0:1767319610.587478:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319610.587484:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319610.587546:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89180 x1853163985212544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319610.587563:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319610.587566:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319610.587567:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319610.587570:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4853 00000100:00080000:0.0:1767319610.587573:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319610.587576:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319610.587587:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89880 x1853163985212672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319610.587620:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319610.587624:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319610.587626:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319610.587629:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4853 00000100:00080000:0.0:1767319610.587632:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319610.587635:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319610.587646:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88a80 x1853163985212928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319610.587670:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319610.587674:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319610.587675:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319610.587679:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4853 00000100:00080000:0.0:1767319610.587681:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319610.587684:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319610.587692:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a680 x1853163985213184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319610.587724:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985213312:0@lo:400:kworker.0 00000100:00100000:0.0:1767319610.587787:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319610.587847:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319610.587852:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985213056:0@lo:400:kworker.0 00000100:00100000:0.0:1767319610.587877:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319610.587910:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213312 00000100:00100000:0.0:1767319610.587915:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213312 00000100:00100000:0.0:1767319610.587966:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3095 00000100:00100000:0.0:1767319610.587975:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985213312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319610.588020:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985213312:12345-0@lo:400:kworker.0 Request processed in 47us (232us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319610.588027:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3095 00000100:00100000:0.0:1767319610.588072:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985213312:0@lo:400:kworker.0 00000100:00080000:0.0:1767319610.588091:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319610.588182:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985213440:0@lo:400:kworker.0 00000100:00100000:1.0:1767319610.588212:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319610.588242:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319610.588248:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985212800:0@lo:400:kworker.0 00000100:00100000:1.0:1767319610.588268:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319610.588299:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213056 00000100:00100000:1.0:1767319610.588303:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213056 00000100:00100000:1.0:1767319610.588332:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212800 00000100:00100000:1.0:1767319610.588335:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212800 00000100:00100000:1.0:1767319610.588352:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4722 00000100:00100000:1.0:1767319610.588357:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985213056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319610.588390:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985213056:12345-0@lo:400:kworker.0 Request processed in 31us (511us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319610.588396:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4722 00000100:00100000:1.0:1767319610.588409:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4723 00000100:00100000:1.0:1767319610.588415:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985212800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319610.588446:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985212800:12345-0@lo:400:kworker.0 Request processed in 31us (177us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319610.588452:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4723 00000100:00100000:1.0:1767319610.588480:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985212800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319610.588662:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985212544:0@lo:38:kworker.0 00000100:00100000:2.0:1767319610.588691:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319610.588719:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985212672:0@lo:38:kworker.0 00000100:00100000:2.0:1767319610.588740:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319610.588765:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985212928:0@lo:38:kworker.0 00000100:00100000:2.0:1767319610.588788:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319610.588810:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985213184:0@lo:38:kworker.0 00000100:00100000:2.0:1767319610.588832:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319610.588866:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212544 00000100:00100000:2.0:1767319610.588870:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212544 00000100:00100000:2.0:1767319610.588888:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212928 00000100:00100000:2.0:1767319610.588890:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212928 00000100:00100000:2.0:1767319610.588897:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213184 00000100:00100000:2.0:1767319610.588899:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213184 00000100:00100000:2.0:1767319610.588908:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5232 00000100:00100000:2.0:1767319610.588913:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985212544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319610.588930:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319610.588964:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985212544:12345-0@lo:38:kworker.0 Request processed in 50us (272us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319610.588970:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5232 00000100:00100000:2.0:1767319610.588983:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5233 00000100:00100000:2.0:1767319610.588986:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985212928:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319610.588995:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319610.589015:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985212928:12345-0@lo:38:kworker.0 Request processed in 27us (227us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319610.589020:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5233 00000100:00100000:2.0:1767319610.589029:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5234 00000100:00100000:2.0:1767319610.589032:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985213184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319610.589040:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319610.589058:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985213184:12345-0@lo:38:kworker.0 Request processed in 24us (226us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319610.589063:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5234 00000100:00100000:2.0:1767319610.589079:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985212672 00000100:00100000:2.0:1767319610.589082:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985212672 00000100:00100000:2.0:1767319610.589098:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1798 00000100:00100000:2.0:1767319610.589102:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985212672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319610.589110:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319610.589132:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985212672:12345-0@lo:38:kworker.0 Request processed in 29us (391us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319610.589137:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1798 00000100:00080000:2.0:1767319610.589188:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319610.589193:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319610.589196:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985212544:0@lo:38:kworker.0 00000100:00080000:2.0:1767319610.589205:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319610.589208:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319610.589211:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985212672:0@lo:38:kworker.0 00000100:00080000:2.0:1767319610.589217:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319610.589219:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319610.589222:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985212928:0@lo:38:kworker.0 00000100:00080000:2.0:1767319610.589228:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319610.589231:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319610.589233:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985213184:0@lo:38:kworker.0 00000100:00100000:0.0:1767319610.589488:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213440 00000100:00100000:0.0:1767319610.589493:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213440 00000100:00100000:0.0:1767319610.590594:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3096 00000100:00100000:0.0:1767319610.590607:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985213440:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319610.590673:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985213440:12345-0@lo:400:kworker.0 Request processed in 70us (2459us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319610.590680:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3096 00000100:00100000:2.0:1767319610.590765:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985213440:0@lo:400:kworker.0 00000100:00100000:3.0:1767319610.593994:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985213056:0@lo:400:kworker.0 00000100:00080000:3.0:1767319610.594032:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319610.594038:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319611.228326:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319612.123432:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985213568:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319612.123526:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319612.123699:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213568 00000100:00100000:0.0:1767319612.123707:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213568 00000100:00100000:0.0:1767319612.123778:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2272 00000100:00100000:0.0:1767319612.123788:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985213568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319612.123818:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319612.123864:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985213568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (337us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319612.123871:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2272 00000100:00100000:0.0:1767319612.123915:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985213568:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319612.123990:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985213696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319612.124019:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319612.124109:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213696 00000100:00100000:2.0:1767319612.124114:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213696 00000100:00080000:1.0:1767319612.124116:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319612.124125:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319612.124152:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2273 00000100:00100000:2.0:1767319612.124161:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985213696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319612.124196:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319612.124238:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985213696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (217us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319612.124245:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2273 00000100:00100000:1.0:1767319612.124292:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985213696:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319612.251365:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319613.275414:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319614.299416:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319614.637839:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767319614.637872:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920570496 offset 0 length 520: 4 00000100:00100000:1.0:1767319614.637897:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319614.637989:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920570752 00000100:00100000:2.0:1767319614.638001:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920570752 00000100:00100000:2.0:1767319614.638083:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920570624 00000100:00100000:2.0:1767319614.638086:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920570624 00000100:00100000:2.0:1767319614.638107:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4724 00000100:00100000:2.0:1767319614.638129:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920570752:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319614.638228:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920570752:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 112us (399us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319614.638235:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4724 00000100:00100000:2.0:1767319614.638252:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5235 00000100:00100000:2.0:1767319614.638257:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920570624:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319614.638275:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319614.638294:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920570624:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 36us (397us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319614.638301:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5235 00000100:00080000:1.0:1767319614.638626:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319615.323676:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319615.707415:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319615.707435:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319615.707437:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319615.707441:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319615.707446:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319615.707454:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4858 00000100:00080000:0.0:1767319615.707458:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319615.707466:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319615.707507:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb800 x1853163985213824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319615.707524:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319615.707527:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319615.707529:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319615.707532:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4858 00000100:00080000:0.0:1767319615.707534:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319615.707537:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319615.707551:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b480 x1853163985213952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319615.707595:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319615.707599:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319615.707601:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319615.707605:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4858 00000100:00080000:0.0:1767319615.707607:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319615.707610:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319615.707625:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88a80 x1853163985214208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319615.707651:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319615.707654:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319615.707656:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319615.707659:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4858 00000100:00080000:0.0:1767319615.707662:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319615.707666:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319615.707677:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89180 x1853163985214464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319615.707730:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985214592:0@lo:400:kworker.0 00000100:00100000:0.0:1767319615.707799:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319615.707813:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985214080:0@lo:400:kworker.0 00000100:00100000:3.0:1767319615.707862:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319615.707867:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319615.707877:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985214336:0@lo:400:kworker.0 00000100:00100000:3.0:1767319615.707896:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319615.707904:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985214720:0@lo:400:kworker.0 00000100:00100000:0.0:1767319615.707911:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319615.707933:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319615.707953:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985214592 00000100:00100000:0.0:1767319615.707962:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985214592 00000100:00100000:3.0:1767319615.707969:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985214080 00000100:00100000:3.0:1767319615.707974:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985214080 00000100:00100000:1.0:1767319615.708043:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985214720 00000100:00100000:1.0:1767319615.708049:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985214720 00000100:00100000:0.0:1767319615.708124:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3097 00000100:00100000:0.0:1767319615.708133:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985214592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319615.708197:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985214336 00000100:00100000:3.0:1767319615.708201:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985214336 00000100:00100000:0.0:1767319615.708214:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985214592:12345-0@lo:400:kworker.0 Request processed in 81us (414us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319615.708223:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3097 00000100:00100000:3.0:1767319615.708229:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4725 00000100:00100000:3.0:1767319615.708237:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985214080:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319615.708256:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4726 00000100:00100000:0.0:1767319615.708266:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985214336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319615.708280:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985214080:12345-0@lo:400:kworker.0 Request processed in 43us (417us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319615.708288:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4725 00000100:00100000:0.0:1767319615.708306:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985214336:12345-0@lo:400:kworker.0 Request processed in 41us (395us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319615.708313:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4726 00000100:00100000:3.0:1767319615.708340:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985214080:0@lo:400:kworker.0 00000100:00100000:0.0:1767319615.708358:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985214336:0@lo:400:kworker.0 00000100:00100000:0.0:1767319615.708371:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985214592:0@lo:400:kworker.0 00000100:00080000:3.0:1767319615.708379:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319615.708385:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319615.708790:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319590/real 1767319590] req@ffff8ce1b5b8b100 x1853163985207296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319615 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319615.708810:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5b8b100 x1853163985207296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319615 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319615.708824:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985213824:0@lo:38:kworker.0 00000100:00100000:2.0:1767319615.708855:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319615.708893:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985213952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319615.708918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319615.708944:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985214208:0@lo:38:kworker.0 00000100:00100000:2.0:1767319615.708962:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319615.708973:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985214464:0@lo:38:kworker.0 00000100:00100000:2.0:1767319615.709012:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319615.709027:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319615.709042:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319615.709045:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985207296:0@lo:250:kworker.0 00000100:00080000:2.0:1767319615.709070:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319615.709074:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1767319615.709151:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213952 00000100:00100000:3.0:1767319615.709156:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213952 00000100:00100000:3.0:1767319615.709196:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1799 00000100:00100000:3.0:1767319615.709202:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985213952:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319615.709219:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319615.709257:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985213952:12345-0@lo:38:kworker.0 Request processed in 54us (337us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319615.709261:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319615.709266:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1799 00000100:00080000:2.0:1767319615.709270:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4838 00000100:00080000:2.0:1767319615.709274:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319615.709279:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319615.709287:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985213824 00000100:00100000:3.0:1767319615.709292:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985213824 00000100:00100000:2.0:1767319615.709299:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7800 x1853163985214848/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319615.709309:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985214208 00000100:00100000:3.0:1767319615.709313:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985214208 00000100:00100000:3.0:1767319615.709335:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985214464 00000100:00100000:3.0:1767319615.709338:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985214464 00000100:00100000:2.0:1767319615.709342:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5236 00000100:00100000:2.0:1767319615.709348:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985213824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319615.709415:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319615.709453:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985213824:12345-0@lo:38:kworker.0 Request processed in 103us (597us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319615.709812:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5237 00000100:00100000:3.0:1767319615.709818:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985214208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319615.709828:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319615.712059:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3098 00000100:00100000:1.0:1767319615.712075:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985214720:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319615.712402:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985214720:12345-0@lo:400:kworker.0 Request processed in 332us (4468us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319615.712410:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3098 00000100:00080000:1.0:1767319615.712454:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319615.712496:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985214720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319615.712553:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5238 00000100:00100000:1.0:1767319615.712559:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985214464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319615.712576:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319615.712608:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985214464:12345-0@lo:38:kworker.0 Request processed in 47us (3602us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319615.712614:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5238 00000100:00100000:1.0:1767319615.712639:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985214848:0@lo:250:kworker.0 00000400:00000100:1.0:1767319615.712664:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985214848 offset 0 length 520: 4 00000100:00080000:1.0:1767319615.712696:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319615.712701:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319615.712704:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985213824:0@lo:38:kworker.0 00000100:00080000:1.0:1767319615.712713:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319615.712716:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319615.712719:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985213952:0@lo:38:kworker.0 00000100:00080000:1.0:1767319615.712728:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319615.712731:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319615.712733:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985214464:0@lo:38:kworker.0 00000100:00100000:3.0:1767319615.713647:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985214208:12345-0@lo:38:kworker.0 Request processed in 3824us (4680us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319615.713656:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5237 00000100:00080000:1.0:1767319615.713944:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319615.713949:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319615.713953:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985214208:0@lo:38:kworker.0 00000100:00100000:2.0:1767319615.716271:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5236 00010000:00010000:1.0:1767319616.347589:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319617.244512:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319617.244519:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985215104:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319617.244531:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985214976:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319617.250665:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319617.251005:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985214976 00000100:00100000:1.0:1767319617.251017:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985214976 00000100:00100000:1.0:1767319617.251107:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2274 00000100:00100000:1.0:1767319617.251117:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985214976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319617.251152:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319617.251235:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985214976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 120us (579us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319617.251243:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2274 00000100:00100000:1.0:1767319617.251292:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985214976:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319617.251540:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319617.255414:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319617.255957:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985215104 00000100:00100000:2.0:1767319617.255967:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985215104 00000100:00100000:2.0:1767319617.256040:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2275 00000100:00100000:2.0:1767319617.256050:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985215104:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319617.256083:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319617.256147:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985215104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (739us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319617.256155:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2275 00000100:00080000:3.0:1767319617.256238:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319617.256261:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985215104:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319617.371730:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319618.395887:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319619.420891:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319619.753513:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319619.753890:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571008 00000100:00100000:0.0:1767319619.753906:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571008 00000100:00100000:0.0:1767319619.754012:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4727 00000100:00100000:0.0:1767319619.754025:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920571008:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319619.754128:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920571008:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (630us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319619.754137:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4727 00000100:00100000:2.0:1767319619.754365:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:3.0:1767319619.754438:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319619.754786:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920570880 00000100:00100000:0.0:1767319619.754793:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920570880 00000100:00100000:0.0:1767319619.754831:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5239 00000100:00100000:0.0:1767319619.754838:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920570880:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319619.754863:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319619.754916:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920570880:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 76us (552us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319619.754924:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5239 00010000:00010000:1.0:1767319620.445090:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319620.828890:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319620.828905:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319620.828909:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319620.828913:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319620.828919:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319620.828930:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4863 00000100:00080000:3.0:1767319620.828943:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319620.828952:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319620.829008:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89180 x1853163985215232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319620.829044:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319620.829048:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319620.829050:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319620.829054:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4863 00000100:00080000:3.0:1767319620.829057:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319620.829061:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319620.829083:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b480 x1853163985215360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319620.829136:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319620.829140:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319620.829142:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319620.829146:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4863 00000100:00080000:3.0:1767319620.829148:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319620.829151:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319620.829192:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b100 x1853163985215616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319620.829220:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319620.829223:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319620.829225:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319620.829229:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4863 00000100:00080000:3.0:1767319620.829232:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319620.829236:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319620.829247:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89880 x1853163985215872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319620.829383:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985215488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319620.829476:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319620.829568:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319620.829575:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985216128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319620.829605:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319620.829765:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985215488 00000100:00100000:3.0:1767319620.829776:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985215488 00000100:00100000:3.0:1767319620.829857:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4728 00000100:00100000:3.0:1767319620.829867:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985215488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319620.829937:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985215488:12345-0@lo:400:kworker.0 Request processed in 71us (459us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319620.829945:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4728 00000100:00100000:1.0:1767319620.830872:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216128 00000100:00100000:1.0:1767319620.830882:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216128 00000100:00100000:1.0:1767319620.830954:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3099 00000100:00100000:1.0:1767319620.830965:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985216128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319620.831038:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985216128:12345-0@lo:400:kworker.0 Request processed in 75us (1431us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319620.831046:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3099 00000100:00100000:1.0:1767319620.831081:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985216000:0@lo:400:kworker.0 00000100:00100000:1.0:1767319620.831121:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319620.831155:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319620.831201:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985215744:0@lo:400:kworker.0 00000100:00100000:1.0:1767319620.831230:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319620.831279:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985215232:0@lo:38:kworker.0 00000100:00100000:1.0:1767319620.831307:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319620.831344:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985215360:0@lo:38:kworker.0 00000100:00100000:1.0:1767319620.831374:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319620.831401:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985215616:0@lo:38:kworker.0 00000100:00100000:1.0:1767319620.831426:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319620.831440:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985215872:0@lo:38:kworker.0 00000100:00100000:1.0:1767319620.831454:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319620.831485:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985215360 00000100:00100000:1.0:1767319620.831490:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985215360 00000100:00100000:3.0:1767319620.831744:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216000 00000100:00100000:3.0:1767319620.831748:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216000 00000100:00100000:3.0:1767319620.831772:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3100 00000100:00100000:3.0:1767319620.831777:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985216000:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319620.831815:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985216000:12345-0@lo:400:kworker.0 Request processed in 39us (695us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319620.831819:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3100 00000100:00100000:3.0:1767319620.831835:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985215744 00000100:00100000:3.0:1767319620.831837:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985215744 00000100:00100000:3.0:1767319620.831859:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985215232 00000100:00100000:3.0:1767319620.831862:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985215232 00000100:00100000:3.0:1767319620.831871:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985215616 00000100:00100000:3.0:1767319620.831873:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985215616 00000100:00100000:3.0:1767319620.831880:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985215872 00000100:00100000:3.0:1767319620.831882:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985215872 00000100:00100000:3.0:1767319620.831890:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4729 00000100:00100000:3.0:1767319620.831896:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985215744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319620.831926:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985215744:12345-0@lo:400:kworker.0 Request processed in 30us (695us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319620.831932:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4729 00000100:00100000:3.0:1767319620.831944:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5240 00000100:00100000:3.0:1767319620.831948:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985215232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319620.831971:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319620.832007:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985215232:12345-0@lo:38:kworker.0 Request processed in 57us (699us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319620.832015:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5240 00000100:00100000:3.0:1767319620.832026:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5241 00000100:00100000:3.0:1767319620.832029:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985215616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319620.832036:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319620.832053:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985215616:12345-0@lo:38:kworker.0 Request processed in 23us (627us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319620.832056:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5241 00000100:00100000:3.0:1767319620.832062:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5242 00000100:00100000:3.0:1767319620.832065:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985215872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319620.832070:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319620.832083:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985215872:12345-0@lo:38:kworker.0 Request processed in 18us (628us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319620.832087:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5242 00000100:00080000:3.0:1767319620.832118:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319620.832122:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319620.832125:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985215232:0@lo:38:kworker.0 00000100:00080000:3.0:1767319620.832131:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319620.832134:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319620.832136:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985215616:0@lo:38:kworker.0 00000100:00080000:3.0:1767319620.832140:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319620.832143:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319620.832144:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985215872:0@lo:38:kworker.0 00000100:00100000:3.0:1767319620.832228:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1800 00000100:00100000:3.0:1767319620.832232:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985215360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319620.832238:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319620.832259:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985215360:12345-0@lo:38:kworker.0 Request processed in 26us (885us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319620.832262:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1800 00000100:00080000:3.0:1767319620.832282:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319620.832284:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319620.832286:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985215360:0@lo:38:kworker.0 00000100:00100000:3.0:1767319620.832307:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985215744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319620.832313:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985216000:0@lo:400:kworker.0 00000100:00080000:0.0:1767319620.832495:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319620.832503:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319620.832507:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319620.832547:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985216128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319620.832560:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985215488:0@lo:400:kworker.0 00010000:00010000:2.0:1767319621.468468:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319622.364107:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319622.364133:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319622.364135:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985216384:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319622.364150:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985216256:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319622.365120:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319622.366267:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216384 00000100:00100000:2.0:1767319622.366572:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216384 00000100:00100000:2.0:1767319622.366898:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2276 00000100:00100000:2.0:1767319622.366911:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb5850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985216384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319622.366947:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319622.367036:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb5850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985216384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 128us (1922us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319622.367044:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2276 00000100:00100000:1.0:1767319622.367675:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985216384:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319622.369531:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319622.377687:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319622.377843:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216256 00000100:00100000:2.0:1767319622.377858:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216256 00000100:00100000:2.0:1767319622.377951:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2277 00000100:00100000:2.0:1767319622.377962:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985216256:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319622.378007:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767319622.378642:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319622.379438:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985216256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1474us (1761us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319622.379458:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2277 00000100:00100000:0.0:1767319622.379759:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985216256:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319622.491638:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319623.515595:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319624.540320:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319624.876088:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319624.878514:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571264 00000100:00100000:0.0:1767319624.878528:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571264 00000100:00080000:3.0:1767319624.878769:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319624.878866:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4730 00000100:00100000:0.0:1767319624.878877:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920571264:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319624.880152:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920571264:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 1266us (4064us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319624.880180:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4730 00000100:00100000:2.0:1767319624.888871:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319624.889581:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571136 00000100:00100000:0.0:1767319624.889596:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571136 00000100:00100000:0.0:1767319624.889744:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5243 00000100:00100000:0.0:1767319624.889754:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920571136:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319624.889783:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319624.889893:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920571136:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 139us (1031us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319624.889901:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5243 00010000:00010000:1.0:1767319625.567015:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319625.947341:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319625.947354:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319625.947358:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319625.947367:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319625.947372:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319625.947383:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4869 00000100:00080000:0.0:1767319625.947390:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319625.947397:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319625.947440:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163985216512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319625.947471:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319625.947475:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319625.947477:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319625.947480:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4869 00000100:00080000:0.0:1767319625.947483:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319625.947487:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319625.947500:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75880 x1853163985216640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319625.947543:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319625.947547:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319625.947549:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319625.947552:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4869 00000100:00080000:0.0:1767319625.947554:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319625.947558:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319625.947569:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74700 x1853163985216896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319625.947601:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319625.947605:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319625.947607:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319625.947610:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4869 00000100:00080000:0.0:1767319625.947612:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319625.947615:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319625.947626:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985217152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319625.947673:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985217024:0@lo:400:kworker.0 00000100:00100000:0.0:1767319625.947764:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319625.947836:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319625.947843:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985217280:0@lo:400:kworker.0 00000100:00100000:0.0:1767319625.947872:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319625.947909:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217280 00000100:00100000:0.0:1767319625.947917:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217280 00000100:00100000:0.0:1767319625.947979:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3101 00000100:00100000:0.0:1767319625.947988:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985217280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319625.948044:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985217280:12345-0@lo:400:kworker.0 Request processed in 56us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319625.948052:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3101 00000100:00100000:0.0:1767319625.948096:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985217280:0@lo:400:kworker.0 00000100:00100000:1.0:1767319625.948187:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985216768:0@lo:400:kworker.0 00000100:00100000:1.0:1767319625.948217:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319625.948263:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319625.948269:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985217408:0@lo:400:kworker.0 00000100:00100000:1.0:1767319625.948291:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319625.948315:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217408 00000100:00100000:1.0:1767319625.948318:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217408 00000100:00100000:1.0:1767319625.948343:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3102 00000100:00100000:1.0:1767319625.948350:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985217408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319625.948381:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985217408:12345-0@lo:400:kworker.0 Request processed in 31us (89us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319625.948388:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3102 00000100:00100000:1.0:1767319625.948411:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985217408:0@lo:400:kworker.0 00000100:00100000:3.0:1767319625.948491:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217024 00000100:00100000:3.0:1767319625.948496:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217024 00000100:00100000:3.0:1767319625.948535:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216768 00000100:00100000:3.0:1767319625.948538:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216768 00000100:00100000:3.0:1767319625.948558:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4731 00000100:00100000:3.0:1767319625.948563:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985217024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319625.948600:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985217024:12345-0@lo:400:kworker.0 Request processed in 37us (837us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319625.948607:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4731 00000100:00100000:3.0:1767319625.948619:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4732 00000100:00100000:3.0:1767319625.948625:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985216768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319625.948653:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985216768:12345-0@lo:400:kworker.0 Request processed in 28us (436us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319625.948658:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4732 00000100:00080000:3.0:1767319625.948685:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319625.948689:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319625.948692:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319625.948703:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985216512:0@lo:38:kworker.0 00000100:00100000:3.0:1767319625.948728:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319625.948744:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985216640:0@lo:38:kworker.0 00000100:00100000:3.0:1767319625.948765:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319625.948787:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985216896:0@lo:38:kworker.0 00000100:00100000:3.0:1767319625.948803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319625.948816:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985217152:0@lo:38:kworker.0 00000100:00100000:3.0:1767319625.948832:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319625.948858:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216512 00000100:00100000:3.0:1767319625.948861:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216512 00000100:00100000:3.0:1767319625.948878:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216896 00000100:00100000:3.0:1767319625.948881:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216896 00000100:00100000:3.0:1767319625.948888:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217152 00000100:00100000:3.0:1767319625.948890:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217152 00000100:00100000:3.0:1767319625.948898:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5244 00000100:00100000:3.0:1767319625.948902:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985216512:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319625.948921:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319625.948953:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985216512:12345-0@lo:38:kworker.0 Request processed in 50us (224us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319625.948959:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5244 00000100:00100000:3.0:1767319625.948970:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5245 00000100:00100000:3.0:1767319625.948973:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985216896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319625.948982:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319625.949002:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985216896:12345-0@lo:38:kworker.0 Request processed in 27us (198us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319625.949008:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5245 00000100:00100000:3.0:1767319625.949018:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5246 00000100:00100000:3.0:1767319625.949021:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985217152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319625.949029:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319625.949050:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985217152:12345-0@lo:38:kworker.0 Request processed in 27us (216us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319625.949055:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5246 00000100:00080000:3.0:1767319625.949083:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319625.949089:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319625.949092:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985216512:0@lo:38:kworker.0 00000100:00080000:3.0:1767319625.949101:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319625.949104:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319625.949107:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985216896:0@lo:38:kworker.0 00000100:00080000:3.0:1767319625.949113:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319625.949116:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319625.949118:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985217152:0@lo:38:kworker.0 00000100:00100000:3.0:1767319625.949150:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985217024:0@lo:400:kworker.0 00000100:00100000:1.0:1767319625.949317:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985216640 00000100:00100000:1.0:1767319625.949322:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985216640 00000100:00100000:1.0:1767319625.949352:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1801 00000100:00100000:1.0:1767319625.949357:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985216640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319625.949369:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319625.949408:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985216640:12345-0@lo:38:kworker.0 Request processed in 49us (641us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319625.949413:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1801 00000100:00100000:1.0:1767319625.949440:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985216768:0@lo:400:kworker.0 00000100:00080000:2.0:1767319625.949506:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319625.949511:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319625.949514:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985216640:0@lo:38:kworker.0 00010000:00010000:1.0:1767319626.588095:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319627.483657:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319627.483685:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985217536:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319627.483754:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319627.483919:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217536 00000100:00100000:0.0:1767319627.483929:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217536 00000100:00100000:0.0:1767319627.483995:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2278 00000100:00100000:0.0:1767319627.484004:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985217536:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319627.484032:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319627.484078:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985217536:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319627.484085:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2278 00000100:00100000:0.0:1767319627.484123:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985217536:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319627.484304:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319627.484419:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319627.484429:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985217664:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319627.484459:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319627.484742:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217664 00000100:00100000:2.0:1767319627.484747:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217664 00000100:00100000:2.0:1767319627.484786:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2279 00000100:00100000:2.0:1767319627.484794:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985217664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319627.484819:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319627.484850:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985217664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (390us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319627.484855:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2279 00000100:00080000:1.0:1767319627.485205:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319627.485244:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985217664:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319627.611527:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319628.635422:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319629.660237:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319629.994905:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319629.995243:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571392 00000100:00100000:0.0:1767319629.995254:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571392 00000100:00100000:0.0:1767319629.995335:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5247 00000100:00100000:0.0:1767319629.995344:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920571392:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319629.995367:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319629.995436:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920571392:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (549us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319629.995443:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5247 00000100:00100000:2.0:1767319629.995749:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319629.995925:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571520 00000100:00100000:1.0:1767319629.995930:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571520 00000100:00100000:1.0:1767319629.995979:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4733 00000100:00100000:1.0:1767319629.995986:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920571520:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319629.996036:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920571520:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 51us (287us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319629.996042:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4733 00000100:00080000:2.0:1767319629.996556:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319630.683537:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319631.067554:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319631.067566:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319631.067568:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319631.067572:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319631.067577:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319631.067585:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4874 00000100:00080000:3.0:1767319631.067590:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319631.067597:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319631.067641:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89180 x1853163985217792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319631.067677:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319631.067681:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319631.067683:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319631.067686:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4874 00000100:00080000:3.0:1767319631.067689:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319631.067693:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319631.067707:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a680 x1853163985217920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319631.067740:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319631.067745:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319631.067746:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319631.067751:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4874 00000100:00080000:3.0:1767319631.067754:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319631.067757:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319631.067768:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985218176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319631.067798:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319631.067801:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319631.067803:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319631.067806:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4874 00000100:00080000:3.0:1767319631.067808:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319631.067811:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319631.067822:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88a80 x1853163985218432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319631.067855:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985218560:0@lo:400:kworker.0 00000100:00100000:3.0:1767319631.067917:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319631.067974:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319631.067980:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985218304:0@lo:400:kworker.0 00000100:00100000:3.0:1767319631.068005:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319631.068037:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218560 00000100:00100000:3.0:1767319631.068043:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218560 00000100:00100000:3.0:1767319631.068093:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3103 00000100:00100000:3.0:1767319631.068101:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985218560:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319631.068331:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985218560:12345-0@lo:400:kworker.0 Request processed in 229us (412us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319631.068339:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3103 00000100:00080000:3.0:1767319631.068361:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319631.068393:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985218560:0@lo:400:kworker.0 00000100:00100000:3.0:1767319631.068425:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218304 00000100:00100000:2.0:1767319631.068643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985217792:0@lo:38:kworker.0 00000100:00100000:2.0:1767319631.068685:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319631.068708:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985217920:0@lo:38:kworker.0 00000100:00100000:2.0:1767319631.068731:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319631.068751:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985218176:0@lo:38:kworker.0 00000100:00100000:2.0:1767319631.068768:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319631.068782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985218432:0@lo:38:kworker.0 00000100:00100000:2.0:1767319631.068801:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319631.068829:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217792 00000100:00100000:2.0:1767319631.068833:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217792 00000100:00100000:2.0:1767319631.068856:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218176 00000100:00100000:2.0:1767319631.068858:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218176 00000100:00100000:2.0:1767319631.068872:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218432 00000100:00100000:2.0:1767319631.068874:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218432 00000100:00100000:2.0:1767319631.068884:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5248 00000100:00100000:2.0:1767319631.068889:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985217792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319631.068906:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319631.068936:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985217792:12345-0@lo:38:kworker.0 Request processed in 46us (250us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319631.068943:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5248 00000100:00100000:2.0:1767319631.068956:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5249 00000100:00100000:2.0:1767319631.068960:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985218176:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319631.068969:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319631.068990:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985218176:12345-0@lo:38:kworker.0 Request processed in 28us (221us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319631.068996:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5249 00000100:00100000:2.0:1767319631.069008:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5250 00000100:00100000:2.0:1767319631.069011:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985218432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319631.069020:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319631.069040:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985218432:12345-0@lo:38:kworker.0 Request processed in 27us (238us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319631.069045:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5250 00000100:00080000:2.0:1767319631.069070:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319631.069075:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319631.069078:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985217792:0@lo:38:kworker.0 00000100:00080000:2.0:1767319631.069086:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319631.069089:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319631.069091:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985218176:0@lo:38:kworker.0 00000100:00080000:2.0:1767319631.069097:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319631.069100:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319631.069102:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985218432:0@lo:38:kworker.0 00000100:00100000:3.0:1767319631.069294:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218304 00000100:00100000:3.0:1767319631.069334:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4734 00000100:00100000:3.0:1767319631.069341:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985218304:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319631.069380:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985218304:12345-0@lo:400:kworker.0 Request processed in 40us (1373us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319631.069386:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4734 00000100:00080000:3.0:1767319631.069401:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319631.069412:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985217920 00000100:00100000:3.0:1767319631.069416:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985217920 00000100:00100000:0.0:1767319631.069883:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985218688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319631.069917:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319631.069943:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319631.069949:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985218048:0@lo:400:kworker.0 00000100:00100000:0.0:1767319631.069968:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319631.069990:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218048 00000100:00100000:0.0:1767319631.069993:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218048 00000100:00100000:0.0:1767319631.070023:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4735 00000100:00100000:0.0:1767319631.070029:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985218048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319631.070058:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985218048:12345-0@lo:400:kworker.0 Request processed in 29us (89us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319631.070063:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4735 00000100:00100000:0.0:1767319631.070080:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218688 00000100:00100000:0.0:1767319631.070083:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218688 00000100:00100000:0.0:1767319631.070106:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3104 00000100:00100000:0.0:1767319631.070123:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985218688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319631.070178:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985218688:12345-0@lo:400:kworker.0 Request processed in 38us (243us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319631.070185:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3104 00000100:00080000:0.0:1767319631.070314:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319631.070317:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319631.070337:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985218048:0@lo:400:kworker.0 00000100:00100000:0.0:1767319631.070345:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985218688:0@lo:400:kworker.0 00000100:00100000:3.0:1767319631.070563:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1802 00000100:00100000:3.0:1767319631.070569:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985217920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319631.070582:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319631.070615:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985217920:12345-0@lo:38:kworker.0 Request processed in 45us (1883us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319631.070620:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1802 00000100:00100000:2.0:1767319631.070683:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985218304:0@lo:400:kworker.0 00000100:00080000:0.0:1767319631.071536:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319631.071542:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319631.071545:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985217920:0@lo:38:kworker.0 00010000:00010000:1.0:1767319631.711221:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319632.605323:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319632.605345:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985218816:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319632.605353:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319632.605364:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985218944:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319632.607235:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319632.609048:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218944 00000100:00100000:3.0:1767319632.609063:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218944 00000100:00100000:3.0:1767319632.609312:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2280 00000100:00100000:3.0:1767319632.609325:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985218944:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319632.609359:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319632.609444:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985218944:12345-0@lo:13:osp-pre-0-1.0 Request processed in 122us (2212us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319632.609451:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2280 00000100:00100000:0.0:1767319632.609547:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985218944:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319632.610103:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319632.610400:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985218816 00000100:00100000:1.0:1767319632.610406:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985218816 00000100:00100000:1.0:1767319632.610451:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2281 00000100:00100000:1.0:1767319632.610460:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985218816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319632.610481:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319632.610529:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985218816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (428us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319632.610536:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2281 00000100:00100000:1.0:1767319632.610571:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985218816:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319632.731396:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319633.755962:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319634.783367:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319635.116367:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319635.116420:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319635.118428:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571648 00000100:00100000:0.0:1767319635.118447:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571648 00000100:00100000:0.0:1767319635.118524:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571776 00000100:00100000:0.0:1767319635.118528:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571776 00000100:00100000:0.0:1767319635.118599:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4736 00000100:00100000:0.0:1767319635.118611:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920571776:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319635.118701:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920571776:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 92us (2280us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319635.118709:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4736 00000100:00100000:0.0:1767319635.118729:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5251 00000100:00100000:0.0:1767319635.118734:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920571648:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319635.118753:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319635.118779:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920571648:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (2424us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319635.118784:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5251 00000100:00080000:3.0:1767319635.119474:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319635.803790:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319636.187397:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319636.187411:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319636.187415:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319636.187420:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319636.187425:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319636.187436:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4879 00000100:00080000:0.0:1767319636.187442:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319636.187451:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319636.187499:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630380 x1853163985219072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319636.187520:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319636.187524:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319636.187526:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319636.187529:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4879 00000100:00080000:0.0:1767319636.187531:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319636.187535:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319636.187547:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630a80 x1853163985219200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319636.187593:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319636.187597:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319636.187599:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319636.187603:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4879 00000100:00080000:0.0:1767319636.187606:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319636.187609:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319636.187622:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc56d80 x1853163985219456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319636.187653:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319636.187657:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319636.187658:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319636.187662:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4879 00000100:00080000:0.0:1767319636.187664:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319636.187668:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319636.187680:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18d500 x1853163985219712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319636.187767:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985219584:0@lo:400:kworker.0 00000100:00100000:0.0:1767319636.187836:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319636.187903:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319636.187909:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985219840:0@lo:400:kworker.0 00000100:00100000:0.0:1767319636.187938:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319636.187969:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219840 00000100:00100000:0.0:1767319636.187976:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219840 00000100:00100000:0.0:1767319636.188039:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3105 00000100:00100000:0.0:1767319636.188078:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985219840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319636.188138:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985219840:12345-0@lo:400:kworker.0 Request processed in 89us (199us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319636.188146:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3105 00000100:00100000:0.0:1767319636.188221:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985219840:0@lo:400:kworker.0 00000100:00100000:2.0:1767319636.188312:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985219072:0@lo:38:kworker.0 00000100:00100000:2.0:1767319636.188343:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319636.188374:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985219200:0@lo:38:kworker.0 00000100:00100000:2.0:1767319636.188396:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319636.188419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985219456:0@lo:38:kworker.0 00000100:00100000:2.0:1767319636.188438:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319636.188453:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985219712:0@lo:38:kworker.0 00000100:00100000:2.0:1767319636.188471:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319636.188499:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219584 00000100:00100000:2.0:1767319636.188503:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219584 00000100:00100000:2.0:1767319636.188527:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219072 00000100:00100000:2.0:1767319636.188530:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219072 00000100:00100000:2.0:1767319636.188541:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219456 00000100:00100000:2.0:1767319636.188544:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219456 00000100:00100000:2.0:1767319636.188551:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219712 00000100:00100000:2.0:1767319636.188553:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219712 00000100:00100000:2.0:1767319636.188561:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4737 00000100:00100000:2.0:1767319636.188568:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985219584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319636.188602:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985219584:12345-0@lo:400:kworker.0 Request processed in 35us (766us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319636.188609:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4737 00000100:00100000:2.0:1767319636.188622:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5252 00000100:00100000:2.0:1767319636.188627:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985219072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319636.188644:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319636.188671:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985219072:12345-0@lo:38:kworker.0 Request processed in 43us (328us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319636.188678:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5252 00000100:00100000:2.0:1767319636.188687:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5253 00000100:00100000:2.0:1767319636.188692:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985219456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319636.188701:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319636.188720:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985219456:12345-0@lo:38:kworker.0 Request processed in 27us (281us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319636.188727:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5253 00000100:00100000:2.0:1767319636.188736:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5254 00000100:00100000:2.0:1767319636.188740:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985219712:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319636.188749:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319636.188768:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985219712:12345-0@lo:38:kworker.0 Request processed in 26us (296us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319636.188774:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5254 00000100:00100000:2.0:1767319636.188789:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219200 00000100:00100000:2.0:1767319636.188808:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219200 00000100:00100000:2.0:1767319636.188825:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1803 00000100:00100000:2.0:1767319636.188829:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985219200:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319636.188838:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319636.188860:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985219200:12345-0@lo:38:kworker.0 Request processed in 30us (463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319636.188866:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1803 00000100:00080000:2.0:1767319636.188898:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319636.188903:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319636.188906:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985219072:0@lo:38:kworker.0 00000100:00080000:2.0:1767319636.188914:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319636.188918:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319636.188920:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985219200:0@lo:38:kworker.0 00000100:00080000:2.0:1767319636.188927:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319636.188930:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319636.188933:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985219456:0@lo:38:kworker.0 00000100:00080000:2.0:1767319636.188939:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319636.188942:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319636.188945:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985219712:0@lo:38:kworker.0 00000100:00100000:2.0:1767319636.188978:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985219584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319636.189011:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985219328:0@lo:400:kworker.0 00000100:00100000:3.0:1767319636.189044:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319636.189112:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985219968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319636.189145:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319636.189186:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219328 00000100:00100000:3.0:1767319636.189191:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219328 00000100:00080000:2.0:1767319636.189205:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319636.189212:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319636.189232:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4738 00000100:00100000:3.0:1767319636.189243:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985219328:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319636.189295:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985219328:12345-0@lo:400:kworker.0 Request processed in 55us (250us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319636.189302:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4738 00000100:00100000:0.0:1767319636.189421:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985219968 00000100:00100000:0.0:1767319636.189426:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985219968 00000100:00100000:0.0:1767319636.189464:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3106 00000100:00100000:0.0:1767319636.189472:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985219968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319636.189507:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985219968:12345-0@lo:400:kworker.0 Request processed in 37us (363us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319636.189514:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3106 00000100:00100000:0.0:1767319636.189539:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985219968:0@lo:400:kworker.0 00000100:00080000:3.0:1767319636.189808:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319636.190008:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985219328:0@lo:400:kworker.0 00010000:00010000:2.0:1767319636.827414:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319637.723718:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319637.723745:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985220096:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319637.723823:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319637.724001:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985220096 00000100:00100000:0.0:1767319637.724009:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985220096 00000100:00100000:0.0:1767319637.724074:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2282 00000100:00100000:0.0:1767319637.724084:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985220096:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319637.724116:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319637.724184:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985220096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319637.724193:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2282 00000100:00100000:0.0:1767319637.724247:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985220096:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319637.725194:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319637.725210:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985220224:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319637.725257:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319637.725468:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985220224 00000100:00100000:2.0:1767319637.725476:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985220224 00000100:00100000:2.0:1767319637.725521:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2283 00000100:00100000:2.0:1767319637.725531:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985220224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319637.725555:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319637.725604:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985220224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (345us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319637.725613:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2283 00000100:00100000:1.0:1767319637.725687:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985220224:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319637.851663:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319638.876011:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319639.899456:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319640.238297:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319640.238358:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767319640.238387:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920572160 offset 0 length 520: 4 00000100:00100000:3.0:1767319640.238636:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920571904 00000100:00100000:3.0:1767319640.238653:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920571904 00000100:00100000:3.0:1767319640.238715:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920572032 00000100:00100000:3.0:1767319640.238718:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920572032 00000100:00100000:3.0:1767319640.238768:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4739 00000100:00100000:3.0:1767319640.238778:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920572032:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319640.238876:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920572032:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (517us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319640.238885:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4739 00000100:00100000:3.0:1767319640.238906:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5255 00000100:00100000:3.0:1767319640.238910:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920571904:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319640.238934:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319640.238961:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920571904:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (681us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319640.238967:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5255 00000100:00080000:3.0:1767319640.238997:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319640.283431:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319615/real 1767319615] req@ffff8ce19dca7800 x1853163985214848/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319640 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319640.283470:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19dca7800 x1853163985214848/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319640 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319640.283484:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319640.283507:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319640.283511:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985214848:0@lo:250:kworker.0 00000100:00080000:2.0:1767319640.283616:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319640.283621:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319640.283625:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319640.283629:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4863 00000100:00080000:2.0:1767319640.283634:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319640.283641:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319640.283682:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc56d80 x1853163985220352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319640.283711:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319640.283715:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319640.283717:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319640.283721:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4884 00000100:00080000:2.0:1767319640.283724:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319640.283728:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319640.283748:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631500 x1853163985220480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319640.283757:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319640.283761:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319640.283763:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319640.283766:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4884 00000100:00080000:2.0:1767319640.283769:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319640.283772:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319640.283785:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630a80 x1853163985220608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319640.283849:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985220736:0@lo:400:kworker.0 00000100:00100000:1.0:1767319640.283929:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319640.284065:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985220736 00000100:00100000:3.0:1767319640.284077:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985220736 00000100:00100000:3.0:1767319640.284151:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4740 00000100:00100000:3.0:1767319640.284189:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985220736:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319640.284215:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319640.284221:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319640.284224:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319640.284229:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4884 00000100:00080000:2.0:1767319640.284232:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319640.284237:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319640.284254:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0e00 x1853163985220864/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319640.284279:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319640.284282:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319640.284283:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319640.284285:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4884 00000100:00080000:2.0:1767319640.284287:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319640.284289:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319640.284296:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405500 x1853163985221120/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319640.284416:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985220352:0@lo:250:kworker.0 00000400:00000100:0.0:1767319640.284438:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985220352 offset 0 length 520: 4 00000100:00100000:0.0:1767319640.284455:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985220480:0@lo:38:kworker.0 00000100:00100000:0.0:1767319640.284471:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.284483:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985220608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319640.284499:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.284510:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985220864:0@lo:38:kworker.0 00000100:00100000:0.0:1767319640.284521:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.284530:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985221120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319640.284541:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.284569:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985220608 00000100:00100000:0.0:1767319640.284572:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985220608 00000100:00100000:0.0:1767319640.284591:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1804 00000100:00100000:0.0:1767319640.284595:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985220608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319640.284616:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319640.284642:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985220608:12345-0@lo:38:kworker.0 Request processed in 45us (141us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319640.284648:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1804 00000100:00100000:0.0:1767319640.284667:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985220480 00000100:00100000:0.0:1767319640.284669:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985220480 00000100:00100000:0.0:1767319640.284681:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985220864 00000100:00100000:0.0:1767319640.284682:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985220864 00000100:00100000:0.0:1767319640.284687:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985221120 00000100:00100000:0.0:1767319640.284688:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985221120 00000100:00100000:0.0:1767319640.284696:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5256 00000100:00100000:0.0:1767319640.284700:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985220480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319640.284709:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319640.284734:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985220480:12345-0@lo:38:kworker.0 Request processed in 32us (261us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319640.284739:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5256 00000100:00100000:0.0:1767319640.284751:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5257 00000100:00100000:0.0:1767319640.284755:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985220864:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319640.284763:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319640.284792:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985220864:12345-0@lo:38:kworker.0 Request processed in 36us (269us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319640.284797:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5257 00000100:00100000:0.0:1767319640.284808:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5258 00000100:00100000:0.0:1767319640.284811:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985221120:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319640.284818:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319640.284837:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985221120:12345-0@lo:38:kworker.0 Request processed in 25us (295us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319640.284842:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5258 00000100:00080000:0.0:1767319640.284886:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319640.284892:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319640.284895:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985220480:0@lo:38:kworker.0 00000100:00080000:0.0:1767319640.284903:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319640.284905:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319640.284908:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985220608:0@lo:38:kworker.0 00000100:00080000:0.0:1767319640.284915:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319640.284918:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319640.284920:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985220864:0@lo:38:kworker.0 00000100:00080000:0.0:1767319640.284926:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319640.284929:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319640.284931:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985221120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319640.285002:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985221248:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285032:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.285057:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319640.285063:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985220992:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285082:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.285113:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985221248 00000100:00100000:0.0:1767319640.285117:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985221248 00000100:00100000:0.0:1767319640.285151:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3107 00000100:00100000:0.0:1767319640.285157:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985221248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285221:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985221248:12345-0@lo:400:kworker.0 Request processed in 64us (189us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319640.285228:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3107 00000100:00100000:0.0:1767319640.285249:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985220992 00000100:00100000:0.0:1767319640.285254:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985220992 00000100:00100000:0.0:1767319640.285280:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4741 00000100:00100000:0.0:1767319640.285286:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985220992:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285317:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985220992:12345-0@lo:400:kworker.0 Request processed in 31us (233us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319640.285323:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4741 00000100:00100000:0.0:1767319640.285343:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985221376:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285367:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.285396:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985221376 00000100:00100000:0.0:1767319640.285400:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985221376 00000100:00100000:0.0:1767319640.285424:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3108 00000100:00100000:0.0:1767319640.285429:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985221376:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285463:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985221376:12345-0@lo:400:kworker.0 Request processed in 34us (95us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319640.285467:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3108 00000100:00100000:0.0:1767319640.285488:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985221376:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285502:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985220992:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.285507:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985221248:0@lo:400:kworker.0 00000100:00080000:0.0:1767319640.285564:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319640.285569:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319640.285572:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319640.290846:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985220736:12345-0@lo:400:kworker.0 Request processed in 6666us (6895us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319640.290868:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4740 00000100:00100000:1.0:1767319640.291003:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985220736:0@lo:400:kworker.0 00010000:00010000:1.0:1767319640.924598:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319641.947547:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319642.843862:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985221632:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319642.843973:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319642.844091:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985221632 00000100:00100000:2.0:1767319642.844110:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985221632 00000100:00100000:2.0:1767319642.844221:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2284 00000100:00100000:2.0:1767319642.844246:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985221632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319642.844290:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319642.844356:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985221632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 126us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319642.844365:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2284 00000100:00080000:0.0:1767319642.844683:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319642.844734:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985221632:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319642.844990:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985221504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319642.845034:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319642.845202:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985221504 00000100:00100000:2.0:1767319642.845209:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985221504 00000100:00080000:0.0:1767319642.845380:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319642.845489:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2285 00000100:00100000:2.0:1767319642.845500:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985221504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319642.845528:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319642.845580:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985221504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (545us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319642.845588:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2285 00000100:00100000:1.0:1767319642.845670:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985221504:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319642.974677:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319643.995971:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319645.020062:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319645.353629:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319645.354123:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920572416 00000100:00100000:3.0:1767319645.354135:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920572416 00000100:00100000:3.0:1767319645.354286:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4742 00000100:00100000:3.0:1767319645.354297:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920572416:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767319645.354981:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319645.355135:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920572416:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 838us (1526us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319645.355146:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4742 00000100:00100000:1.0:1767319645.355367:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319645.356200:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920572288 00000100:00100000:3.0:1767319645.356211:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920572288 00000100:00100000:2.0:1767319645.356329:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5259 00000100:00100000:2.0:1767319645.356337:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920572288:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319645.356359:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319645.356530:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920572288:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 189us (1161us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319645.356537:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5259 00000100:00080000:1.0:1767319645.403888:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319645.403898:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319645.403902:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319645.403906:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319645.403910:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319645.403921:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4888 00000100:00080000:1.0:1767319645.403927:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319645.403936:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319645.403980:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e2680 x1853163985221760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319645.404005:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319645.404009:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319645.404010:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319645.404014:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4888 00000100:00080000:1.0:1767319645.404016:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319645.404020:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319645.404035:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0e00 x1853163985221888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319645.404119:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985222016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319645.404216:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319645.404335:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222016 00000100:00100000:2.0:1767319645.404344:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222016 00000100:00100000:2.0:1767319645.404414:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4743 00000100:00100000:2.0:1767319645.404423:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985222016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.404547:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985221760:0@lo:38:kworker.0 00000100:00100000:0.0:1767319645.404577:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.404597:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985221888:0@lo:38:kworker.0 00000100:00100000:0.0:1767319645.404628:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.404684:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985221760 00000100:00100000:0.0:1767319645.404687:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985221760 00000100:00100000:0.0:1767319645.404710:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5260 00000100:00100000:0.0:1767319645.404727:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985221760:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319645.404745:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319645.404780:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985221760:12345-0@lo:38:kworker.0 Request processed in 63us (202us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319645.404786:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5260 00000100:00100000:0.0:1767319645.404818:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985221888 00000100:00100000:0.0:1767319645.404822:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985221888 00000100:00100000:0.0:1767319645.404841:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1805 00000100:00100000:0.0:1767319645.404845:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985221888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319645.404855:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319645.404887:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985221888:12345-0@lo:38:kworker.0 Request processed in 31us (250us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319645.404892:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1805 00000100:00080000:0.0:1767319645.405082:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319645.405089:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319645.405093:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985221760:0@lo:38:kworker.0 00000100:00080000:0.0:1767319645.405101:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319645.405105:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319645.405107:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985221888:0@lo:38:kworker.0 00000100:00080000:0.0:1767319645.405150:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319645.408454:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319645.408465:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319645.408470:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319645.408478:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4888 00000100:00080000:1.0:1767319645.408483:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319645.408489:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319645.408530:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985222144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319645.408573:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319645.408576:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319645.408578:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319645.408581:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4888 00000100:00080000:1.0:1767319645.408583:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319645.408586:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319645.408596:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1500 x1853163985222400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319645.408751:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985222656:0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.408816:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.408897:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222656 00000100:00100000:0.0:1767319645.408905:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222656 00000100:00100000:0.0:1767319645.408958:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3109 00000100:00100000:0.0:1767319645.408966:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985222656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.409016:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985222656:12345-0@lo:400:kworker.0 Request processed in 51us (200us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319645.409023:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3109 00000100:00080000:0.0:1767319645.409044:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319645.409075:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985222656:0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.409111:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985222272:0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.409138:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.409160:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319645.409188:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985222528:0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.409209:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.409233:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985222144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319645.409256:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.409272:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985222400:0@lo:38:kworker.0 00000100:00100000:0.0:1767319645.409291:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.409319:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222272 00000100:00100000:0.0:1767319645.409322:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222272 00000100:00100000:0.0:1767319645.409349:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222144 00000100:00100000:0.0:1767319645.409351:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222144 00000100:00100000:0.0:1767319645.409361:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222400 00000100:00100000:0.0:1767319645.409363:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222400 00000100:00100000:0.0:1767319645.409372:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4744 00000100:00100000:0.0:1767319645.409377:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985222272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.410096:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985222272:12345-0@lo:400:kworker.0 Request processed in 715us (954us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319645.410105:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4744 00000100:00100000:0.0:1767319645.410126:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5261 00000100:00100000:0.0:1767319645.410131:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985222144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319645.410150:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319645.410207:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985222144:12345-0@lo:38:kworker.0 Request processed in 74us (949us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319645.410213:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5261 00000100:00100000:0.0:1767319645.410224:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5262 00000100:00100000:0.0:1767319645.410228:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985222400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319645.410237:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319645.410261:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985222400:12345-0@lo:38:kworker.0 Request processed in 31us (968us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319645.410266:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5262 00000100:00100000:0.0:1767319645.410301:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222528 00000100:00100000:0.0:1767319645.410305:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222528 00000100:00100000:0.0:1767319645.410334:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3110 00000100:00100000:0.0:1767319645.410341:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985222528:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.410374:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985222528:12345-0@lo:400:kworker.0 Request processed in 34us (1164us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319645.410380:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3110 00000100:00080000:0.0:1767319645.412688:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319645.412699:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319645.412829:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319645.412838:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319645.412843:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985222144:0@lo:38:kworker.0 00000100:00080000:0.0:1767319645.412856:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319645.412859:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319645.412862:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985222400:0@lo:38:kworker.0 00000100:00100000:0.0:1767319645.412924:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985222528:0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.412936:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985222272:0@lo:400:kworker.0 00000100:00100000:2.0:1767319645.415488:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985222016:12345-0@lo:400:kworker.0 Request processed in 11057us (11262us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319645.415504:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4743 00000100:00100000:3.0:1767319645.415635:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985222016:0@lo:400:kworker.0 00010000:00010000:2.0:1767319646.044702:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319647.067361:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319647.963693:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319647.963724:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985222784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319647.963806:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319647.963969:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222784 00000100:00100000:2.0:1767319647.963980:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222784 00000100:00100000:2.0:1767319647.964060:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2286 00000100:00100000:2.0:1767319647.964072:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985222784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319647.964104:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319647.964153:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985222784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (346us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319647.964188:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2286 00000100:00100000:2.0:1767319647.964234:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985222784:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319647.964408:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319647.964434:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319647.964450:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985222912:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319647.964483:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319647.964561:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985222912 00000100:00100000:3.0:1767319647.964567:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985222912 00000100:00100000:3.0:1767319647.964610:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2287 00000100:00100000:3.0:1767319647.964617:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985222912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319647.964635:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319647.964676:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985222912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (193us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319647.964683:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2287 00000100:00080000:1.0:1767319647.964733:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319647.964746:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985222912:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319648.091385:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319649.115396:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319650.139519:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319650.474604:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319650.474706:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319650.474822:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920572672 00000100:00100000:2.0:1767319650.474834:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920572672 00000100:00100000:2.0:1767319650.474905:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920572544 00000100:00100000:2.0:1767319650.474909:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920572544 00000100:00100000:2.0:1767319650.474927:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4745 00000100:00100000:2.0:1767319650.474937:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920572672:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319650.475015:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920572672:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (425us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319650.475022:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4745 00000100:00100000:2.0:1767319650.475041:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5263 00000100:00100000:2.0:1767319650.475045:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920572544:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319650.475063:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319650.475085:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920572544:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 39us (379us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319650.475091:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5263 00000100:00080000:1.0:1767319650.475576:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319650.523675:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319650.523687:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319650.523690:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319650.523696:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319650.523702:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319650.523713:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4893 00000100:00080000:1.0:1767319650.523719:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319650.523727:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319650.523773:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1500 x1853163985223040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319650.523816:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319650.523820:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319650.523822:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319650.523826:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4893 00000100:00080000:1.0:1767319650.523828:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319650.523832:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319650.523848:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1f80 x1853163985223168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319650.523889:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319650.523893:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319650.523895:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319650.523899:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4893 00000100:00080000:1.0:1767319650.523902:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319650.523905:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319650.523917:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594a80 x1853163985223424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319650.523946:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319650.523950:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319650.523952:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319650.523955:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4893 00000100:00080000:1.0:1767319650.523958:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319650.523961:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319650.523973:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5597480 x1853163985223680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319650.524015:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985223808:0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.524097:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319650.524188:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319650.524196:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985223552:0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.524225:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319650.524264:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223552 00000100:00100000:1.0:1767319650.524271:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223552 00000100:00100000:1.0:1767319650.524323:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4746 00000100:00100000:1.0:1767319650.524331:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985223552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.524380:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985223552:12345-0@lo:400:kworker.0 Request processed in 49us (154us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319650.524387:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4746 00000100:00080000:1.0:1767319650.524408:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319650.524438:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985223552:0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.524830:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985223936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.524866:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319650.524899:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319650.524905:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985223296:0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.524928:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319650.524959:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223296 00000100:00100000:2.0:1767319650.524998:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223296 00000100:00100000:2.0:1767319650.525037:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4747 00000100:00100000:2.0:1767319650.525043:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985223296:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.525292:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985223296:12345-0@lo:400:kworker.0 Request processed in 247us (362us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319650.525299:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4747 00000100:00080000:2.0:1767319650.525318:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319650.525337:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985223296:0@lo:400:kworker.0 00000100:00100000:0.0:1767319650.525494:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223808 00000100:00100000:0.0:1767319650.525500:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223808 00000100:00100000:0.0:1767319650.525550:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223936 00000100:00100000:0.0:1767319650.525554:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223936 00000100:00100000:0.0:1767319650.525578:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3111 00000100:00100000:0.0:1767319650.525587:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985223808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319650.525632:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985223808:12345-0@lo:400:kworker.0 Request processed in 47us (1534us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319650.525639:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3111 00000100:00100000:0.0:1767319650.525653:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3112 00000100:00100000:0.0:1767319650.525659:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985223936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319650.525689:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985223936:12345-0@lo:400:kworker.0 Request processed in 30us (823us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319650.525695:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3112 00000100:00100000:0.0:1767319650.525728:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985223040:0@lo:38:kworker.0 00000100:00100000:0.0:1767319650.525758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319650.525796:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985223168:0@lo:38:kworker.0 00000100:00100000:0.0:1767319650.525822:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319650.525846:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985223424:0@lo:38:kworker.0 00000100:00100000:0.0:1767319650.525866:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319650.525881:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985223680:0@lo:38:kworker.0 00000100:00100000:0.0:1767319650.525900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319650.525930:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223168 00000100:00100000:0.0:1767319650.525934:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223168 00000100:00100000:0.0:1767319650.525956:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1806 00000100:00100000:0.0:1767319650.525960:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985223168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319650.525979:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319650.526011:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985223168:12345-0@lo:38:kworker.0 Request processed in 49us (188us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319650.526017:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1806 00000100:00100000:0.0:1767319650.526036:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223040 00000100:00100000:0.0:1767319650.526040:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223040 00000100:00100000:0.0:1767319650.526055:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223424 00000100:00100000:0.0:1767319650.526058:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223424 00000100:00100000:3.0:1767319650.526407:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985223680 00000100:00100000:3.0:1767319650.526412:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985223680 00000100:00100000:3.0:1767319650.526436:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5264 00000100:00100000:3.0:1767319650.526441:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985223040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319650.526452:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319650.526497:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985223040:12345-0@lo:38:kworker.0 Request processed in 55us (738us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319650.526503:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5264 00000100:00100000:3.0:1767319650.526516:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5265 00000100:00100000:3.0:1767319650.526520:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985223424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319650.526529:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319650.526553:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985223424:12345-0@lo:38:kworker.0 Request processed in 31us (685us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319650.526557:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5265 00000100:00100000:3.0:1767319650.526568:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5266 00000100:00100000:3.0:1767319650.526571:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985223680:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319650.526580:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319650.526603:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985223680:12345-0@lo:38:kworker.0 Request processed in 30us (702us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319650.526608:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5266 00000100:00080000:3.0:1767319650.526651:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319650.526655:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319650.526659:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985223040:0@lo:38:kworker.0 00000100:00080000:3.0:1767319650.526670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319650.526673:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319650.526676:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985223168:0@lo:38:kworker.0 00000100:00080000:3.0:1767319650.526683:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319650.526687:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319650.526689:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985223424:0@lo:38:kworker.0 00000100:00080000:3.0:1767319650.526697:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319650.526700:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319650.526702:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985223680:0@lo:38:kworker.0 00000100:00080000:2.0:1767319650.526838:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319650.526843:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319650.526865:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985223936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.530674:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985223808:0@lo:400:kworker.0 00010000:00010000:1.0:1767319651.163783:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319652.192011:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319653.083571:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985224064:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319653.083694:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319653.083863:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985224192:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319653.083898:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319653.083965:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224064 00000100:00100000:2.0:1767319653.083976:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224064 00000100:00100000:2.0:1767319653.084044:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224192 00000100:00100000:2.0:1767319653.084047:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224192 00000100:00100000:2.0:1767319653.084074:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2288 00000100:00100000:2.0:1767319653.084086:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985224064:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319653.084128:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319653.084257:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2289 00000100:00100000:1.0:1767319653.084268:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985224192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319653.084289:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319653.084334:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985224192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (434us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319653.084343:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2289 00000100:00080000:1.0:1767319653.084402:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319653.084408:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319653.084545:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985224192:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319653.085554:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985224064:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1463us (1852us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319653.085567:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2288 00000100:00100000:3.0:1767319653.085850:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985224064:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319653.212569:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319654.242707:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319655.259774:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319655.594114:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319655.594917:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920572928 00000100:00100000:0.0:1767319655.594936:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920572928 00000100:00080000:1.0:1767319655.595005:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319655.595048:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4748 00000100:00100000:0.0:1767319655.595062:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920572928:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319655.595234:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920572928:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 176us (1137us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319655.595243:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4748 00000100:00100000:3.0:1767319655.596152:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319655.596966:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920572800 00000100:00100000:2.0:1767319655.596981:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920572800 00000100:00100000:2.0:1767319655.597049:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5267 00000100:00100000:2.0:1767319655.597059:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920572800:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319655.597092:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319655.597206:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920572800:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 147us (1060us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319655.597215:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5267 00000100:00080000:0.0:1767319655.649880:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.649893:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319655.649897:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.649902:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.649908:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.649919:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.649925:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.649933:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319655.649991:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c000 x1853163985224320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319655.650027:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.650031:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.650033:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.650037:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.650039:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.650043:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319655.650058:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98e00 x1853163985224448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319655.650109:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.650113:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.650115:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.650120:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.650122:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.650126:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319655.650142:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9500 x1853163985224704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319655.650188:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.650191:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.650193:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.650197:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.650200:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.650203:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319655.650213:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bea300 x1853163985224960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319655.650287:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985224576:0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.650373:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319655.650454:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319655.650460:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985225216:0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.650484:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319655.650517:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225216 00000100:00100000:1.0:1767319655.650526:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225216 00000100:00100000:1.0:1767319655.650589:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3113 00000100:00100000:1.0:1767319655.650597:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985225216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.650650:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985225216:12345-0@lo:400:kworker.0 Request processed in 53us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319655.650658:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3113 00000100:00080000:1.0:1767319655.650681:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319655.650704:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985224832:0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.650730:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319655.650750:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319655.650755:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985225088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.650777:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319655.650798:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225088 00000100:00100000:1.0:1767319655.650802:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225088 00000100:00100000:1.0:1767319655.650832:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3114 00000100:00100000:1.0:1767319655.650837:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985225088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.650872:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985225088:12345-0@lo:400:kworker.0 Request processed in 34us (94us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319655.650878:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3114 00000100:00080000:1.0:1767319655.650896:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319655.650930:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985225088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.650959:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985225216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319655.651061:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985224320:0@lo:38:kworker.0 00000100:00100000:3.0:1767319655.651095:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319655.651127:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985224448:0@lo:38:kworker.0 00000100:00100000:3.0:1767319655.651211:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319655.651242:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985224704:0@lo:38:kworker.0 00000100:00100000:3.0:1767319655.651269:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319655.651287:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985224960:0@lo:38:kworker.0 00000100:00100000:3.0:1767319655.651310:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319655.651344:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224448 00000100:00100000:3.0:1767319655.651349:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224448 00000100:00100000:3.0:1767319655.651377:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1807 00000100:00100000:3.0:1767319655.651382:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985224448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.651405:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319655.651442:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985224448:12345-0@lo:38:kworker.0 Request processed in 58us (229us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.651448:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1807 00000100:00100000:3.0:1767319655.651469:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224576 00000100:00100000:3.0:1767319655.651473:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224576 00000100:00100000:3.0:1767319655.651509:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224832 00000100:00100000:3.0:1767319655.651512:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224832 00000100:00100000:3.0:1767319655.651526:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224320 00000100:00100000:3.0:1767319655.651529:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224320 00000100:00100000:3.0:1767319655.651537:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224704 00000100:00100000:3.0:1767319655.651540:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224704 00000100:00100000:3.0:1767319655.651547:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985224960 00000100:00100000:3.0:1767319655.651549:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985224960 00000100:00100000:3.0:1767319655.651558:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4749 00000100:00100000:3.0:1767319655.651564:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985224576:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319655.651606:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985224576:12345-0@lo:400:kworker.0 Request processed in 41us (1232us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319655.651613:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4749 00000100:00100000:3.0:1767319655.651624:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4750 00000100:00100000:3.0:1767319655.651630:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985224832:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319655.651660:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985224832:12345-0@lo:400:kworker.0 Request processed in 30us (929us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319655.651666:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4750 00000100:00100000:3.0:1767319655.651677:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5268 00000100:00100000:3.0:1767319655.651681:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985224320:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.651705:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319655.651730:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985224320:12345-0@lo:38:kworker.0 Request processed in 47us (634us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.651736:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5268 00000100:00100000:3.0:1767319655.651747:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5269 00000100:00100000:3.0:1767319655.651750:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985224704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.651759:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319655.651782:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985224704:12345-0@lo:38:kworker.0 Request processed in 30us (512us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.651788:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5269 00000100:00100000:3.0:1767319655.651798:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5270 00000100:00100000:3.0:1767319655.651801:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985224960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.651810:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319655.651832:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985224960:12345-0@lo:38:kworker.0 Request processed in 29us (521us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.651838:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5270 00000100:00080000:3.0:1767319655.651867:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319655.651888:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985224576:0@lo:400:kworker.0 00000100:00080000:3.0:1767319655.651921:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319655.651927:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319655.651930:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985224320:0@lo:38:kworker.0 00000100:00080000:3.0:1767319655.651939:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319655.651942:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319655.651945:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985224448:0@lo:38:kworker.0 00000100:00080000:3.0:1767319655.651953:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319655.651957:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319655.651959:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985224704:0@lo:38:kworker.0 00000100:00080000:3.0:1767319655.651968:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319655.651972:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319655.651975:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985224960:0@lo:38:kworker.0 00000100:00100000:1.0:1767319655.652125:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985224832:0@lo:400:kworker.0 00010000:00010000:1.0:1767319656.283580:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319657.308019:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319658.203468:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319658.203504:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985225344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319658.203621:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319658.203707:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225344 00000100:00100000:2.0:1767319658.203717:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225344 00000100:00100000:2.0:1767319658.203791:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2290 00000100:00100000:2.0:1767319658.203802:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985225344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319658.203845:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319658.203894:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985225344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (273us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319658.203904:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2290 00000100:00100000:2.0:1767319658.203982:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985225344:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319658.205059:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319658.209247:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319658.210819:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985225472:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319658.211117:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319658.211302:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225472 00000100:00100000:2.0:1767319658.211316:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225472 00000100:00100000:2.0:1767319658.211502:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2291 00000100:00100000:2.0:1767319658.211517:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985225472:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319658.211705:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319658.211781:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985225472:12345-0@lo:13:osp-pre-0-1.0 Request processed in 269us (665us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319658.211790:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2291 00000100:00100000:0.0:1767319658.211920:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985225472:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319658.212110:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319658.333403:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319659.356656:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319660.380282:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319660.715968:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319660.716753:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920573184 00000100:00100000:1.0:1767319660.716769:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920573184 00000100:00100000:1.0:1767319660.716894:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4751 00000100:00100000:3.0:1767319660.717819:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319660.717884:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920573056 00000100:00100000:0.0:1767319660.717897:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920573056 00000100:00100000:0.0:1767319660.717954:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5271 00000100:00100000:0.0:1767319660.717965:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920573056:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319660.717996:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319660.718244:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920573056:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 279us (436us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319660.718253:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5271 00000100:00080000:2.0:1767319660.718636:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319660.718715:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce187feac50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920573184:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319660.718791:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce187feac50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920573184:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 1886us (2887us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319660.718799:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4751 00000100:00080000:2.0:1767319660.763749:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319660.763763:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319660.763767:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319660.763771:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319660.763776:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319660.763786:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4903 00000100:00080000:2.0:1767319660.763790:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319660.763798:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319660.763848:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05500 x1853163985225600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319660.763878:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319660.763881:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319660.763883:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319660.763887:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4903 00000100:00080000:2.0:1767319660.763889:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319660.763892:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319660.763902:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07b80 x1853163985225728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319660.764018:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985225600:0@lo:38:kworker.0 00000100:00100000:3.0:1767319660.764094:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319660.764272:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319660.764276:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319660.764278:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319660.764282:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4903 00000100:00100000:1.0:1767319660.764657:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225600 00000100:00100000:1.0:1767319660.764670:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225600 00000100:00100000:1.0:1767319660.764708:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5272 00000100:00100000:1.0:1767319660.764716:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985225600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319660.764736:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319660.764781:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985225600:12345-0@lo:38:kworker.0 Request processed in 64us (687us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319660.764787:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5272 00000100:00100000:1.0:1767319660.764831:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319660.764841:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985225856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319660.764865:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319660.765000:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985225728:0@lo:38:kworker.0 00000100:00100000:3.0:1767319660.765030:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319660.765059:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225728 00000100:00100000:3.0:1767319660.765063:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225728 00000100:00100000:3.0:1767319660.765086:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1808 00000100:00100000:3.0:1767319660.765091:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985225728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319660.765102:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319660.765128:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985225728:12345-0@lo:38:kworker.0 Request processed in 36us (98us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319660.765134:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1808 00000100:00100000:0.0:1767319660.765245:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225856 00000100:00100000:0.0:1767319660.765249:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225856 00000100:00100000:0.0:1767319660.765301:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4752 00000100:00100000:0.0:1767319660.765308:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985225856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319660.765346:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985225856:12345-0@lo:400:kworker.0 Request processed in 39us (480us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319660.765352:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4752 00000100:00100000:1.0:1767319660.765459:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985225856:0@lo:400:kworker.0 00000100:00080000:2.0:1767319660.765555:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319660.765565:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319660.765606:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985225984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319660.765668:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319660.765673:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319660.765676:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319660.765678:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319660.765682:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4903 00000100:00080000:2.0:1767319660.765689:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319660.765692:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00080000:2.0:1767319660.765698:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319660.765699:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985225728:0@lo:38:kworker.0 00000100:00100000:2.0:1767319660.765721:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05f80 x1853163985226240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319660.765722:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319660.765736:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319660.765751:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985225600:0@lo:38:kworker.0 00000100:00100000:0.0:1767319660.765787:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985225984:0@lo:38:kworker.0 00000100:00100000:0.0:1767319660.765822:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319660.765822:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319660.765845:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985226496:0@lo:400:kworker.0 00000100:00100000:0.0:1767319660.765855:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985226240:0@lo:38:kworker.0 00000100:00100000:1.0:1767319660.766000:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985226112:0@lo:400:kworker.0 00000100:00080000:3.0:1767319660.766062:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319660.766076:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985226368:0@lo:400:kworker.0 00000100:00100000:2.0:1767319660.773032:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319660.773055:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319660.773195:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985225984 00000100:00100000:1.0:1767319660.773207:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985225984 00000100:00100000:2.0:1767319660.773212:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985226496 00000100:00100000:2.0:1767319660.773222:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985226496 00000100:00100000:1.0:1767319660.773248:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985226112 00000100:00100000:1.0:1767319660.773254:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985226112 00000100:00100000:2.0:1767319660.773305:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3115 00000100:00100000:1.0:1767319660.773312:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4753 00000100:00100000:2.0:1767319660.773321:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985226496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319660.773332:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985226112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319660.773432:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985226496:12345-0@lo:400:kworker.0 Request processed in 115us (408us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319660.773450:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3115 00000100:00100000:1.0:1767319660.773452:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985226112:12345-0@lo:400:kworker.0 Request processed in 120us (397us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319660.773465:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4753 00000100:00100000:1.0:1767319660.773502:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5273 00000100:00080000:2.0:1767319660.773502:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319660.773513:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319660.773515:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985225984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319660.773537:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319660.773558:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985226496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319660.773586:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985225984:12345-0@lo:38:kworker.0 Request processed in 66us (7763us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319660.773745:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5273 00000100:00100000:1.0:1767319660.773796:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985226112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319660.774086:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319660.774158:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319660.774253:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319660.774259:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319660.774262:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985225984:0@lo:38:kworker.0 00000100:00100000:1.0:1767319660.774304:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985226240 00000100:00100000:1.0:1767319660.774310:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985226240 00000100:00100000:1.0:1767319660.774386:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5274 00000100:00100000:2.0:1767319660.774389:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985226368 00000100:00100000:1.0:1767319660.774398:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985226240:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319660.774400:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985226368 00010000:00020000:1.0:1767319660.774425:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319660.774468:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3116 00000100:00100000:2.0:1767319660.774482:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985226368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319660.774491:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985226240:12345-0@lo:38:kworker.0 Request processed in 92us (326us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319660.774501:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5274 00000100:00100000:2.0:1767319660.774550:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985226368:12345-0@lo:400:kworker.0 Request processed in 71us (462us total) trans 0 rc 0/0 00000100:00080000:3.0:1767319660.774558:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319660.774560:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3116 00000100:00100000:3.0:1767319660.774609:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985226368:0@lo:400:kworker.0 00000100:00080000:2.0:1767319660.774656:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319660.774662:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319660.774667:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985226240:0@lo:38:kworker.0 00010000:00010000:2.0:1767319661.403363:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319662.427653:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319663.323468:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985226624:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319663.323536:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319663.323714:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319663.323728:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985226752:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319663.324534:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319663.330216:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985226624 00000100:00100000:0.0:1767319663.330233:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985226624 00000100:00100000:0.0:1767319663.330324:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985226752 00000100:00100000:0.0:1767319663.330329:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985226752 00000100:00100000:0.0:1767319663.330369:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2292 00000100:00100000:0.0:1767319663.330379:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985226624:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319663.330411:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319663.330488:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985226624:12345-0@lo:13:osp-pre-0-1.0 Request processed in 111us (6951us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319663.330495:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2292 00000100:00100000:0.0:1767319663.330512:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2293 00000100:00100000:0.0:1767319663.330518:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985226752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319663.330533:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319663.330932:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985226624:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319663.331226:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319663.331230:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319663.331303:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985226752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 784us (6771us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319663.331311:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2293 00000100:00100000:3.0:1767319663.331385:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985226752:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319663.451412:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319664.478044:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319665.499417:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319665.834694:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319665.834747:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319665.834781:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920573568 offset 0 length 520: 4 00000100:00100000:1.0:1767319665.834977:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920573312 00000100:00100000:1.0:1767319665.834990:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920573312 00000100:00100000:1.0:1767319665.835045:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920573440 00000100:00100000:1.0:1767319665.835049:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920573440 00000100:00100000:1.0:1767319665.835126:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4754 00000100:00100000:1.0:1767319665.835138:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920573440:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319665.835280:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920573440:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 145us (532us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319665.835289:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4754 00000100:00100000:1.0:1767319665.835310:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5275 00000100:00100000:1.0:1767319665.835315:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920573312:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319665.835339:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319665.835369:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920573312:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (694us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319665.835376:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5275 00000100:00080000:3.0:1767319665.836831:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319665.886947:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319640/real 1767319640] req@ffff8ce1adc56d80 x1853163985220352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319665 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319665.886991:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1adc56d80 x1853163985220352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319665 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319665.887008:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319665.887042:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319665.887049:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319665.887053:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319665.887058:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4888 00000100:00080000:1.0:1767319665.887206:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319665.887216:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319665.887258:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7480 x1853163985226880/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319665.887276:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319665.887280:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319665.887282:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319665.887286:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4909 00000100:00080000:1.0:1767319665.887289:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319665.887293:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319665.887308:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4000 x1853163985227008/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319665.887316:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319665.887320:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319665.887321:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319665.887325:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4909 00000100:00080000:1.0:1767319665.887327:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319665.887331:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319665.887342:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca5f80 x1853163985227136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319665.887425:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985227264:0@lo:400:kworker.0 00000100:00100000:3.0:1767319665.887531:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319665.887662:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227264 00000100:00100000:0.0:1767319665.887672:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227264 00000100:00100000:0.0:1767319665.887751:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4755 00000100:00100000:0.0:1767319665.887779:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985227264:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319665.887884:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319665.887889:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985220352:0@lo:250:kworker.0 00000100:00100000:2.0:1767319665.887917:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985226880:0@lo:250:kworker.0 00000400:00000100:2.0:1767319665.887943:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985226880 offset 0 length 520: 4 00000100:00100000:2.0:1767319665.887965:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985227008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.887990:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.888010:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985227136:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.888034:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.888083:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227008 00000100:00100000:2.0:1767319665.888087:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227008 00000100:00100000:2.0:1767319665.888126:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5276 00000100:00100000:2.0:1767319665.888132:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985227008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319665.888156:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319665.888212:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985227008:12345-0@lo:38:kworker.0 Request processed in 78us (220us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319665.888220:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5276 00000100:00080000:2.0:1767319665.888263:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.888268:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.888271:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985227008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.888310:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227136 00000100:00100000:2.0:1767319665.888319:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227136 00000100:00100000:2.0:1767319665.888343:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1809 00000100:00100000:2.0:1767319665.888348:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985227136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319665.888358:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319665.888388:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985227136:12345-0@lo:38:kworker.0 Request processed in 39us (352us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319665.888395:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1809 00000100:00080000:2.0:1767319665.888423:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.888427:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.888430:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985227136:0@lo:38:kworker.0 00000100:00080000:2.0:1767319665.888465:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319665.888799:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319665.888803:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319665.888805:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319665.888818:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4909 00000100:00080000:1.0:1767319665.888822:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319665.888826:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319665.888851:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7b80 x1853163985227392/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319665.888884:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319665.888887:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319665.888889:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319665.888893:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4909 00000100:00080000:1.0:1767319665.888895:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319665.888898:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319665.888909:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca5500 x1853163985227648/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319665.888960:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985227904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.888991:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.889019:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985227776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.889034:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.889050:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319665.889054:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985227520:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.889068:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.889085:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227904 00000100:00100000:2.0:1767319665.889088:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227904 00000100:00100000:2.0:1767319665.889115:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227776 00000100:00100000:2.0:1767319665.889127:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227776 00000100:00100000:2.0:1767319665.889140:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3117 00000100:00100000:2.0:1767319665.889146:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985227904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.889188:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985227904:12345-0@lo:400:kworker.0 Request processed in 43us (196us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319665.889193:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3117 00000100:00100000:2.0:1767319665.889201:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3118 00000100:00100000:2.0:1767319665.889205:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985227776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.889222:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985227776:12345-0@lo:400:kworker.0 Request processed in 17us (188us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319665.889226:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3118 00000100:00080000:2.0:1767319665.889238:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319665.889240:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319665.889257:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985227776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.889270:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985227904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.889280:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985227392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.889296:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.889305:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985227648:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.889316:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.889333:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227520 00000100:00100000:2.0:1767319665.889335:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227520 00000100:00100000:2.0:1767319665.889354:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227392 00000100:00100000:2.0:1767319665.889356:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227392 00000100:00100000:2.0:1767319665.889364:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985227648 00000100:00100000:2.0:1767319665.889366:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985227648 00000100:00100000:2.0:1767319665.889371:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4756 00000100:00100000:2.0:1767319665.889375:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985227520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.889396:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985227520:12345-0@lo:400:kworker.0 Request processed in 21us (327us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319665.889400:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4756 00000100:00100000:2.0:1767319665.889407:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5277 00000100:00100000:2.0:1767319665.889409:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985227392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319665.889419:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319665.889437:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985227392:12345-0@lo:38:kworker.0 Request processed in 26us (140us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319665.889442:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5277 00000100:00100000:2.0:1767319665.889679:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5278 00000100:00100000:2.0:1767319665.889684:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985227648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319665.889699:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319665.889727:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985227648:12345-0@lo:38:kworker.0 Request processed in 42us (409us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319665.889734:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5278 00000100:00080000:2.0:1767319665.889755:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319665.889773:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985227520:0@lo:400:kworker.0 00000100:00080000:2.0:1767319665.889804:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.889866:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.889870:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985227392:0@lo:38:kworker.0 00000100:00080000:2.0:1767319665.889884:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.889887:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.889889:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985227648:0@lo:38:kworker.0 00000100:00100000:0.0:1767319665.890090:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985227264:12345-0@lo:400:kworker.0 Request processed in 2313us (2557us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319665.890098:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4755 00000100:00100000:3.0:1767319665.890217:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985227264:0@lo:400:kworker.0 00010000:00010000:1.0:1767319666.523736:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319667.550286:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319668.443819:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985228032:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319668.443917:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319668.444797:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228032 00000100:00100000:2.0:1767319668.444821:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228032 00000100:00100000:2.0:1767319668.444935:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2294 00000100:00100000:2.0:1767319668.444946:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985228032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319668.444987:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767319668.445097:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319668.447865:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985228160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319668.447996:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319668.448893:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228160 00000100:00100000:0.0:1767319668.448917:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228160 00000100:00100000:0.0:1767319668.449013:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2295 00000100:00100000:0.0:1767319668.449027:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985228160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319668.449074:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319668.449549:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985228160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 523us (1551us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319668.449560:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2295 00000100:00100000:0.0:1767319668.449636:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985228160:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319668.450640:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319668.453223:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985228032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 8264us (9291us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319668.453244:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2294 00000100:00100000:1.0:1767319668.454695:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985228032:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319668.571873:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319669.595495:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319670.619637:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319670.953959:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319670.954073:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920573824 00000100:00100000:0.0:1767319670.954093:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920573824 00000100:00100000:0.0:1767319670.954361:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4757 00000100:00100000:0.0:1767319670.954376:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920573824:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319670.954513:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920573824:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 141us (562us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319670.954523:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4757 00000100:00080000:3.0:1767319670.959883:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319670.963425:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319670.963674:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920573696 00000100:00100000:0.0:1767319670.963689:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920573696 00000100:00100000:0.0:1767319670.963825:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5279 00000100:00100000:0.0:1767319670.963836:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920573696:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319670.963865:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319670.963965:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920573696:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 129us (655us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319670.963976:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5279 00000100:00080000:2.0:1767319671.003675:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319671.003689:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319671.003784:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319671.003791:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319671.003797:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319671.003809:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4914 00000100:00080000:2.0:1767319671.003815:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319671.003823:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319671.003883:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985228288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319671.003905:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319671.003909:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319671.003911:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319671.003915:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4914 00000100:00080000:2.0:1767319671.003918:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319671.003922:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319671.003939:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985228416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319671.003997:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985228288:0@lo:38:kworker.0 00000100:00100000:3.0:1767319671.004036:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985228544:0@lo:400:kworker.0 00000100:00100000:0.0:1767319671.004067:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319671.004244:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319671.004249:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319671.004251:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319671.004256:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4914 00000100:00080000:2.0:1767319671.004258:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319671.004263:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319671.004284:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05500 x1853163985228672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319671.004524:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985228416:0@lo:38:kworker.0 00000100:00100000:0.0:1767319671.004548:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319671.004572:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985228672:0@lo:38:kworker.0 00000100:00100000:0.0:1767319671.004585:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319671.004679:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985228800:0@lo:400:kworker.0 00000100:00100000:3.0:1767319671.004892:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228416 00000100:00100000:3.0:1767319671.004904:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228416 00000100:00100000:3.0:1767319671.004959:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1810 00000100:00100000:3.0:1767319671.004968:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985228416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319671.004997:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319671.005053:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985228416:12345-0@lo:38:kworker.0 Request processed in 84us (502us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319671.005061:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1810 00000100:00080000:2.0:1767319671.005488:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319671.005495:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319671.005500:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319671.005505:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4914 00000100:00080000:2.0:1767319671.005508:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319671.005513:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319671.005540:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04e00 x1853163985228928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319671.005953:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985229184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319671.006006:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319671.006122:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985229056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319671.006160:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319671.006532:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229184 00000100:00100000:2.0:1767319671.006543:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229184 00000100:00100000:2.0:1767319671.006850:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229056 00000100:00100000:2.0:1767319671.006855:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229056 00000100:00100000:2.0:1767319671.006885:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3119 00000100:00100000:2.0:1767319671.006894:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985229184:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319671.006959:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985229184:12345-0@lo:400:kworker.0 Request processed in 66us (952us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319671.006967:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3119 00000100:00100000:2.0:1767319671.006985:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3120 00000100:00100000:2.0:1767319671.006991:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985229056:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319671.007020:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985229056:12345-0@lo:400:kworker.0 Request processed in 29us (859us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319671.007025:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3120 00000100:00080000:2.0:1767319671.007050:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319671.007054:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319671.007092:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985229184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319671.007292:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985229056:0@lo:400:kworker.0 00000100:00080000:0.0:1767319671.008655:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319671.008667:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319671.008674:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985228416:0@lo:38:kworker.0 00000100:00100000:0.0:1767319671.008705:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985228928:0@lo:38:kworker.0 00000100:00100000:0.0:1767319671.008747:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319671.010104:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228288 00000100:00100000:1.0:1767319671.010117:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228288 00000100:00100000:1.0:1767319671.010182:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228672 00000100:00100000:1.0:1767319671.010186:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228672 00000100:00100000:1.0:1767319671.010206:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5280 00000100:00100000:1.0:1767319671.010212:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985228288:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319671.010232:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319671.011354:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5281 00000100:00100000:3.0:1767319671.011360:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985228672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319671.011372:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319671.011463:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985228672:12345-0@lo:38:kworker.0 Request processed in 101us (6875us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319671.011470:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5281 00000100:00100000:1.0:1767319671.012226:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985228288:12345-0@lo:38:kworker.0 Request processed in 2012us (8158us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319671.013522:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5280 00000100:00100000:2.0:1767319671.013974:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228928 00000100:00100000:2.0:1767319671.013991:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228928 00000100:00080000:0.0:1767319671.014007:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319671.014014:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319671.014017:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985228672:0@lo:38:kworker.0 00000100:00080000:0.0:1767319671.014028:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319671.014051:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319671.014053:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985228288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319671.014128:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319671.018250:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319671.020045:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228544 00000100:00100000:3.0:1767319671.020058:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228544 00000100:00100000:2.0:1767319671.020091:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5282 00000100:00100000:3.0:1767319671.020147:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4758 00000100:00100000:3.0:1767319671.020159:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985228544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319671.020242:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985228928:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319671.021948:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985228544:12345-0@lo:400:kworker.0 Request processed in 1786us (7813us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319671.021957:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4758 00000100:00080000:3.0:1767319671.021998:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319671.022057:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985228544:0@lo:400:kworker.0 00010000:00020000:2.0:1767319671.022869:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319671.022920:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985228800 00000100:00100000:1.0:1767319671.022927:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985228800 00000100:00100000:2.0:1767319671.022930:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985228928:12345-0@lo:38:kworker.0 Request processed in 2693us (14181us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319671.022937:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5282 00000100:00100000:1.0:1767319671.022971:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4759 00000100:00100000:1.0:1767319671.022979:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985228800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319671.023018:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985228800:12345-0@lo:400:kworker.0 Request processed in 41us (4776us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319671.023021:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4759 00000100:00080000:3.0:1767319671.023063:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319671.023095:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985228800:0@lo:400:kworker.0 00000100:00080000:0.0:1767319671.023119:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319671.023126:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319671.023130:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985228928:0@lo:38:kworker.0 00010000:00010000:1.0:1767319671.645818:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319672.667401:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319673.563487:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319673.563513:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985229312:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319673.563606:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319673.563753:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229312 00000100:00100000:0.0:1767319673.563762:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229312 00000100:00100000:0.0:1767319673.563831:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2296 00000100:00100000:0.0:1767319673.563840:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985229312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319673.563870:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319673.563911:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985229312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (304us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319673.563918:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2296 00000100:00080000:2.0:1767319673.563972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319673.564007:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985229312:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319673.564739:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985229440:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319673.564775:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319673.564825:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229440 00000100:00100000:1.0:1767319673.564832:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229440 00000100:00080000:2.0:1767319673.564885:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319673.564887:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2297 00000100:00100000:1.0:1767319673.564898:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985229440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319673.564919:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319673.564965:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985229440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (188us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319673.564973:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2297 00000100:00100000:2.0:1767319673.565231:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985229440:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319673.692025:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319674.715398:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319675.740371:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319676.077544:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319676.077590:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319676.077651:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920573952 00000100:00100000:1.0:1767319676.077661:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920573952 00000100:00100000:1.0:1767319676.077756:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920574080 00000100:00100000:1.0:1767319676.077760:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920574080 00000100:00080000:0.0:1767319676.077863:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319676.079869:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4760 00000100:00100000:2.0:1767319676.079895:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920574080:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319676.079903:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5283 00000100:00100000:3.0:1767319676.079913:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920573952:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319676.080087:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319676.081132:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920574080:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 1244us (3535us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319676.081142:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4760 00000100:00100000:3.0:1767319676.085186:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920573952:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 5246us (7626us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319676.085202:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5283 00000100:00080000:2.0:1767319676.123704:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319676.123715:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319676.123718:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319676.123723:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319676.123727:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319676.123735:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4919 00000100:00080000:2.0:1767319676.123740:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319676.123748:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319676.123789:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f800 x1853163985229568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319676.123848:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985229568:0@lo:38:kworker.0 00000100:00100000:0.0:1767319676.123915:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319676.124042:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229568 00000100:00100000:3.0:1767319676.124051:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229568 00000100:00100000:3.0:1767319676.124102:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5284 00000100:00100000:3.0:1767319676.124110:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985229568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319676.124131:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319676.125376:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319676.125385:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319676.125389:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319676.125395:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4919 00000100:00080000:2.0:1767319676.125399:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319676.125406:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319676.125445:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c000 x1853163985229696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319676.125621:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985229696:0@lo:38:kworker.0 00000100:00100000:1.0:1767319676.125680:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319676.125777:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985229568:12345-0@lo:38:kworker.0 Request processed in 1665us (1860us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319676.125786:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5284 00000100:00100000:3.0:1767319676.125829:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229696 00000100:00100000:3.0:1767319676.125835:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229696 00000100:00100000:3.0:1767319676.125859:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1811 00000100:00100000:3.0:1767319676.125864:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985229696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319676.125881:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319676.125911:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985229696:12345-0@lo:38:kworker.0 Request processed in 46us (231us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319676.125916:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1811 00000100:00080000:1.0:1767319676.126020:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319676.126025:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319676.126028:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985229568:0@lo:38:kworker.0 00000100:00080000:1.0:1767319676.126050:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319676.126053:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319676.126056:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985229696:0@lo:38:kworker.0 00000100:00100000:0.0:1767319676.127766:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985229824:0@lo:400:kworker.0 00000100:00100000:0.0:1767319676.127824:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319676.127890:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319676.127895:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319676.127898:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319676.127903:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4919 00000100:00080000:2.0:1767319676.127906:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319676.127911:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319676.127930:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985229952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319676.127958:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319676.127962:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319676.127964:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319676.127967:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4919 00000100:00080000:2.0:1767319676.127969:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319676.127972:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319676.127980:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985230208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319676.128049:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229824 00000100:00100000:3.0:1767319676.128057:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229824 00000100:00100000:3.0:1767319676.128104:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4761 00000100:00100000:3.0:1767319676.128113:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985229824:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128200:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985230464:0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128231:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319676.128268:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230464 00000100:00100000:1.0:1767319676.128272:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230464 00000100:00100000:1.0:1767319676.128296:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3121 00000100:00100000:1.0:1767319676.128303:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985230464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128345:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985230464:12345-0@lo:400:kworker.0 Request processed in 43us (112us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319676.128351:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3121 00000100:00100000:1.0:1767319676.128387:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985230464:0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128405:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985229952:0@lo:38:kworker.0 00000100:00100000:1.0:1767319676.128426:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319676.128440:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985230208:0@lo:38:kworker.0 00000100:00100000:1.0:1767319676.128457:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319676.128480:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985229952 00000100:00100000:1.0:1767319676.128482:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985229952 00000100:00100000:1.0:1767319676.128495:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230208 00000100:00100000:1.0:1767319676.128497:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230208 00000100:00100000:1.0:1767319676.128509:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5285 00000100:00100000:1.0:1767319676.128513:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985229952:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319676.128530:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319676.128572:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985229952:12345-0@lo:38:kworker.0 Request processed in 57us (146us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319676.128578:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5285 00000100:00100000:1.0:1767319676.128590:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5286 00000100:00100000:1.0:1767319676.128594:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985230208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319676.128603:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319676.128627:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985230208:12345-0@lo:38:kworker.0 Request processed in 31us (168us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319676.128632:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5286 00000100:00080000:1.0:1767319676.128659:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319676.128663:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319676.128666:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985229952:0@lo:38:kworker.0 00000100:00080000:1.0:1767319676.128674:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319676.128676:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319676.128679:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985230208:0@lo:38:kworker.0 00000100:00080000:1.0:1767319676.128717:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319676.128721:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319676.128750:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985230080:0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128774:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319676.128793:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319676.128798:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985230336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128817:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319676.128835:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230336 00000100:00100000:1.0:1767319676.128839:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230336 00000100:00100000:1.0:1767319676.128868:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3122 00000100:00100000:1.0:1767319676.128874:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985230336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128907:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985230336:12345-0@lo:400:kworker.0 Request processed in 33us (88us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319676.128912:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3122 00000100:00080000:1.0:1767319676.128928:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319676.128942:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985230336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.128957:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230080 00000100:00100000:1.0:1767319676.128960:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230080 00000100:00100000:1.0:1767319676.128986:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4762 00000100:00100000:1.0:1767319676.128990:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985230080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319676.129023:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985230080:12345-0@lo:400:kworker.0 Request processed in 32us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319676.129028:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4762 00000100:00080000:1.0:1767319676.129039:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319676.129052:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985230080:0@lo:400:kworker.0 00000100:00100000:3.0:1767319676.129539:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985229824:12345-0@lo:400:kworker.0 Request processed in 1426us (1713us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319676.129556:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4761 00000100:00100000:1.0:1767319676.129682:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985229824:0@lo:400:kworker.0 00010000:00010000:2.0:1767319676.763588:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319677.787973:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319678.683725:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985230592:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319678.683795:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319678.683894:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230592 00000100:00100000:3.0:1767319678.683902:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230592 00000100:00100000:3.0:1767319678.683962:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2298 00000100:00100000:3.0:1767319678.683972:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985230592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319678.684002:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319678.684043:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985230592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319678.684050:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2298 00000100:00080000:1.0:1767319678.684109:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319678.684148:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985230592:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319678.684531:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319678.684542:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985230720:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319678.684574:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319678.684678:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230720 00000100:00100000:0.0:1767319678.684682:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230720 00000100:00100000:0.0:1767319678.684725:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2299 00000100:00100000:0.0:1767319678.684732:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985230720:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319678.684748:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319678.684792:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985230720:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (216us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319678.684798:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2299 00000100:00100000:2.0:1767319678.684909:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985230720:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319678.684959:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319678.811760:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319679.835907:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319680.860685:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319681.198991:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319681.200044:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920574208 00000100:00100000:3.0:1767319681.200057:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920574208 00000100:00100000:3.0:1767319681.200119:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5287 00000100:00100000:3.0:1767319681.200128:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920574208:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319681.200153:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319681.200269:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920574208:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 141us (1289us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319681.200277:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5287 00000100:00100000:2.0:1767319681.200798:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319681.200857:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920574336 00000100:00100000:1.0:1767319681.200863:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920574336 00000100:00100000:1.0:1767319681.200905:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4763 00000100:00100000:1.0:1767319681.200914:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920574336:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319681.200961:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920574336:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 48us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319681.200968:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4763 00000100:00080000:1.0:1767319681.200988:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319681.246745:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319681.246753:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319681.246757:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319681.246761:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319681.246766:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319681.246775:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4924 00000100:00080000:3.0:1767319681.246780:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319681.246787:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319681.246827:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7100 x1853163985230848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319681.246860:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319681.246864:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319681.246866:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319681.246870:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4924 00000100:00080000:3.0:1767319681.246872:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319681.246876:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319681.246890:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5500 x1853163985230976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319681.246924:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319681.246928:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319681.246930:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319681.246934:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4924 00000100:00080000:3.0:1767319681.246937:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319681.246940:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319681.246951:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163985231232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319681.246975:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319681.246978:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319681.246980:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319681.246984:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4924 00000100:00080000:3.0:1767319681.246986:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319681.246989:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319681.246998:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ce00 x1853163985231488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319681.247460:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985231360:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.247552:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.247621:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319681.247628:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985231616:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.247655:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.247689:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985231360 00000100:00100000:2.0:1767319681.247697:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985231360 00000100:00100000:2.0:1767319681.247751:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4764 00000100:00100000:2.0:1767319681.247760:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985231360:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.247811:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985231360:12345-0@lo:400:kworker.0 Request processed in 51us (258us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319681.247819:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4764 00000100:00100000:2.0:1767319681.247843:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985231616 00000100:00100000:2.0:1767319681.247847:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985231616 00000100:00100000:2.0:1767319681.247874:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3123 00000100:00100000:2.0:1767319681.247882:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985231616:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.247911:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985231616:12345-0@lo:400:kworker.0 Request processed in 30us (255us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319681.247917:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3123 00000100:00080000:2.0:1767319681.247932:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319681.247937:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319681.247972:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985231616:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.247982:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985231360:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.248005:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985231104:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.248028:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.248052:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319681.248058:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985231744:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.248077:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.248099:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985231744 00000100:00100000:2.0:1767319681.248102:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985231744 00000100:00100000:2.0:1767319681.248129:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3124 00000100:00100000:2.0:1767319681.248136:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985231744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.248207:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985231744:12345-0@lo:400:kworker.0 Request processed in 72us (128us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319681.248213:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3124 00000100:00100000:2.0:1767319681.248239:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985231104 00000100:00100000:2.0:1767319681.248242:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985231104 00000100:00100000:2.0:1767319681.248266:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4765 00000100:00100000:2.0:1767319681.248271:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985231104:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.248298:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985231104:12345-0@lo:400:kworker.0 Request processed in 27us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319681.248307:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4765 00000100:00080000:2.0:1767319681.248319:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319681.248321:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319681.248341:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985231744:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.248349:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985231104:0@lo:400:kworker.0 00000100:00100000:2.0:1767319681.248368:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985230848:0@lo:38:kworker.0 00000100:00100000:2.0:1767319681.248390:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.248405:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985230976:0@lo:38:kworker.0 00000100:00100000:2.0:1767319681.248425:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.250963:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985231232:0@lo:38:kworker.0 00000100:00100000:2.0:1767319681.251032:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.251058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985231488:0@lo:38:kworker.0 00000100:00100000:2.0:1767319681.251082:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319681.251139:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230848 00000100:00100000:2.0:1767319681.251147:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230848 00000100:00100000:2.0:1767319681.251309:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985231232 00000100:00100000:2.0:1767319681.251313:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985231232 00000100:00100000:2.0:1767319681.251327:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985231488 00000100:00100000:2.0:1767319681.251329:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985231488 00000100:00100000:2.0:1767319681.251346:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5288 00000100:00100000:2.0:1767319681.251353:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985230848:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319681.251378:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319681.251420:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985230848:12345-0@lo:38:kworker.0 Request processed in 67us (3029us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319681.251427:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5288 00000100:00100000:2.0:1767319681.251446:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5289 00000100:00100000:2.0:1767319681.251450:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985231232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319681.252668:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319681.252727:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985231232:12345-0@lo:38:kworker.0 Request processed in 1274us (1694us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319681.252736:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5289 00000100:00100000:2.0:1767319681.252760:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5290 00000100:00100000:2.0:1767319681.252765:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985231488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319681.252775:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319681.252798:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985231488:12345-0@lo:38:kworker.0 Request processed in 32us (1716us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319681.252804:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5290 00000100:00080000:2.0:1767319681.252880:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319681.252888:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319681.252892:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985230848:0@lo:38:kworker.0 00000100:00080000:2.0:1767319681.252903:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319681.252907:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319681.252909:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985231232:0@lo:38:kworker.0 00000100:00080000:2.0:1767319681.252918:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319681.252921:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319681.252924:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985231488:0@lo:38:kworker.0 00000100:00100000:2.0:1767319681.253001:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985230976 00000100:00100000:2.0:1767319681.253008:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985230976 00000100:00100000:2.0:1767319681.253045:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1812 00000100:00100000:2.0:1767319681.253051:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985230976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319681.253062:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319681.253096:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985230976:12345-0@lo:38:kworker.0 Request processed in 44us (4669us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319681.253102:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1812 00000100:00080000:2.0:1767319681.253125:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319681.253129:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319681.253132:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985230976:0@lo:38:kworker.0 00010000:00010000:1.0:1767319681.883487:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319682.908910:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319683.803763:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319683.803788:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985231872:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319683.803871:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319683.803963:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985231872 00000100:00100000:3.0:1767319683.803972:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985231872 00000100:00100000:3.0:1767319683.804047:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2300 00000100:00100000:3.0:1767319683.804057:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985231872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319683.804088:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319683.804137:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985231872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319683.804145:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2300 00000100:00100000:3.0:1767319683.804217:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985231872:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319683.805211:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319683.806513:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985232000:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319683.806579:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319683.806768:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232000 00000100:00100000:1.0:1767319683.806779:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232000 00000100:00080000:3.0:1767319683.806848:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319683.806862:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2301 00000100:00100000:1.0:1767319683.806874:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985232000:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319683.806905:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319683.810617:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985232000:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3735us (4028us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319683.810632:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2301 00000100:00100000:2.0:1767319683.811018:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985232000:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319683.932155:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319684.958865:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319685.980908:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319686.315207:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319686.315437:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319686.315927:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920574592 00000100:00100000:2.0:1767319686.315940:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920574592 00000100:00100000:3.0:1767319686.316043:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920574464 00000100:00100000:3.0:1767319686.316048:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920574464 00000100:00100000:3.0:1767319686.316084:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4766 00000100:00100000:3.0:1767319686.316095:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920574592:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319686.316228:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920574592:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 135us (1066us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319686.316236:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4766 00000100:00100000:3.0:1767319686.316256:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5291 00000100:00100000:3.0:1767319686.316260:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920574464:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319686.316280:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319686.316305:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920574464:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (869us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319686.316311:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5291 00000100:00080000:0.0:1767319686.318059:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319686.363306:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319686.363318:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319686.363322:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319686.363326:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319686.363337:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319686.363347:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4929 00000100:00080000:3.0:1767319686.363353:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319686.363360:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319686.363399:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73dc00 x1853163985232128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319686.363432:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319686.363436:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319686.363439:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319686.363443:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4929 00000100:00080000:3.0:1767319686.363445:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319686.363447:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319686.363459:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f100 x1853163985232256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319686.363507:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319686.363510:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319686.363517:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319686.363521:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4929 00000100:00080000:3.0:1767319686.363524:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319686.363527:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319686.363541:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c700 x1853163985232512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319686.363568:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319686.363572:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319686.363573:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319686.363576:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4929 00000100:00080000:3.0:1767319686.363579:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319686.363581:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319686.363590:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73d180 x1853163985232768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319686.363631:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985232896:0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.363696:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319686.363775:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319686.363781:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985232640:0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.363805:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319686.364015:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985232384:0@lo:400:kworker.0 00000100:00100000:0.0:1767319686.364044:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319686.364077:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319686.364083:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985233024:0@lo:400:kworker.0 00000100:00100000:0.0:1767319686.364103:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319686.364825:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232640 00000100:00100000:3.0:1767319686.364835:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232640 00000100:00100000:3.0:1767319686.364905:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232384 00000100:00100000:3.0:1767319686.364909:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232384 00000100:00100000:3.0:1767319686.364932:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4767 00000100:00100000:3.0:1767319686.364941:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985232640:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.365002:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985232640:12345-0@lo:400:kworker.0 Request processed in 62us (1195us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319686.365009:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4767 00000100:00100000:3.0:1767319686.365025:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4768 00000100:00100000:3.0:1767319686.365030:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985232384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.365060:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985232384:12345-0@lo:400:kworker.0 Request processed in 30us (1015us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319686.365065:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4768 00000100:00100000:3.0:1767319686.365101:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985232640:0@lo:400:kworker.0 00000100:00100000:2.0:1767319686.365231:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232896 00000100:00100000:2.0:1767319686.365236:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232896 00000100:00100000:2.0:1767319686.365264:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985233024 00000100:00100000:2.0:1767319686.365268:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985233024 00000100:00100000:2.0:1767319686.365300:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3125 00000100:00100000:2.0:1767319686.365308:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985232896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319686.365356:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985232896:12345-0@lo:400:kworker.0 Request processed in 50us (1660us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319686.365362:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3125 00000100:00100000:2.0:1767319686.365376:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3126 00000100:00100000:2.0:1767319686.365381:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985233024:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319686.365406:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985233024:12345-0@lo:400:kworker.0 Request processed in 25us (1302us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319686.365423:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3126 00000100:00100000:2.0:1767319686.365458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985232128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319686.365490:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319686.365601:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232128 00000100:00100000:1.0:1767319686.365605:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232128 00000100:00100000:1.0:1767319686.365630:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5292 00000100:00100000:1.0:1767319686.365635:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985232128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319686.365653:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319686.365684:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985232128:12345-0@lo:38:kworker.0 Request processed in 49us (194us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319686.365690:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5292 00000100:00100000:1.0:1767319686.365739:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985232896:0@lo:400:kworker.0 00000100:00080000:2.0:1767319686.366233:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319686.366238:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319686.366241:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985232128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319686.366248:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985232256:0@lo:38:kworker.0 00000100:00100000:2.0:1767319686.366288:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319686.366317:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985232512:0@lo:38:kworker.0 00000100:00100000:2.0:1767319686.366333:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319686.366355:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985232768:0@lo:38:kworker.0 00000100:00100000:2.0:1767319686.366376:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319686.366468:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319686.366471:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319686.366473:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319686.366492:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985233024:0@lo:400:kworker.0 00000100:00100000:0.0:1767319686.366501:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985232384:0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.366654:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232256 00000100:00100000:3.0:1767319686.366658:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232256 00000100:00100000:3.0:1767319686.366680:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1813 00000100:00100000:3.0:1767319686.366684:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985232256:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319686.366694:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319686.366726:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985232256:12345-0@lo:38:kworker.0 Request processed in 41us (448us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319686.366736:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1813 00000100:00100000:1.0:1767319686.367141:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232512 00000100:00100000:1.0:1767319686.367146:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232512 00000100:00100000:1.0:1767319686.367205:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985232768 00000100:00100000:1.0:1767319686.367208:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985232768 00000100:00100000:1.0:1767319686.367223:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5293 00000100:00100000:1.0:1767319686.367228:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985232512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319686.367240:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319686.367272:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985232512:12345-0@lo:38:kworker.0 Request processed in 43us (938us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319686.367698:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5293 00000100:00100000:1.0:1767319686.367715:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5294 00000100:00100000:1.0:1767319686.367720:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985232768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319686.368076:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319686.368114:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985232768:12345-0@lo:38:kworker.0 Request processed in 393us (1737us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319686.368120:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5294 00000100:00080000:0.0:1767319686.368221:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319686.368227:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319686.368231:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985232256:0@lo:38:kworker.0 00000100:00080000:0.0:1767319686.368240:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319686.368243:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319686.368245:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985232512:0@lo:38:kworker.0 00000100:00080000:0.0:1767319686.368253:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319686.368256:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319686.368258:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985232768:0@lo:38:kworker.0 00010000:00010000:1.0:1767319687.003804:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319688.027430:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319688.923600:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985233152:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319688.923691:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319688.923805:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985233152 00000100:00100000:2.0:1767319688.923814:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985233152 00000100:00100000:2.0:1767319688.923911:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2302 00000100:00100000:2.0:1767319688.923926:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985233152:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319688.923967:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319688.924038:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985233152:12345-0@lo:13:osp-pre-0-1.0 Request processed in 119us (353us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319688.924049:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2302 00000100:00100000:2.0:1767319688.924160:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985233152:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319688.924360:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319688.927443:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985233280:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319688.927559:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319688.927777:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985233280 00000100:00100000:3.0:1767319688.927788:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985233280 00000100:00100000:3.0:1767319688.927889:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2303 00000100:00100000:3.0:1767319688.927904:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985233280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319688.927949:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319688.928026:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985233280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 126us (465us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319688.928036:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2303 00000100:00080000:0.0:1767319688.928122:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319688.928233:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985233280:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319689.053654:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319690.076599:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319690.410413:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319690.410457:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920574720 offset 0 length 520: 4 00000100:00100000:3.0:1767319690.410549:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920574976 00000100:00100000:3.0:1767319690.410569:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920574976 00000100:00100000:2.0:1767319690.410593:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319690.410669:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920574848 00000100:00100000:3.0:1767319690.410676:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920574848 00000100:00100000:1.0:1767319690.410687:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4769 00000100:00100000:1.0:1767319690.410741:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920574976:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319690.410802:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920574976:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 73us (402us total) trans 0 rc 0/0 00000100:00080000:0.0:1767319690.410914:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319690.412155:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5295 00000100:00100000:2.0:1767319690.412200:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920574848:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319690.412224:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319690.412288:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920574848:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 89us (1697us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319690.412296:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5295 00000100:00100000:1.0:1767319690.412356:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4769 00000100:00100000:2.0:1767319690.459466:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319665/real 1767319665] req@ffff8ce19dca7480 x1853163985226880/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319690 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319690.459494:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19dca7480 x1853163985226880/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319690 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319690.459514:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.459542:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319690.459545:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985226880:0@lo:250:kworker.0 00000100:00080000:0.0:1767319690.459626:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319690.459631:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319690.459633:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319690.459637:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4914 00000100:00080000:0.0:1767319690.459640:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319690.459645:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319690.459673:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04e00 x1853163985233408/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319690.459691:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319690.459693:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319690.459695:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319690.459698:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4934 00000100:00080000:0.0:1767319690.459699:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319690.459701:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319690.459710:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07b80 x1853163985233536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319690.459716:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319690.459718:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319690.459719:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319690.459721:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4934 00000100:00080000:0.0:1767319690.459722:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319690.459724:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319690.459730:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06d80 x1853163985233664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319690.459771:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319690.459774:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319690.459776:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319690.459779:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4934 00000100:00080000:0.0:1767319690.459781:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319690.459784:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319690.459796:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05500 x1853163985233920/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319690.459817:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319690.459819:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319690.459820:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319690.459823:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4934 00000100:00080000:0.0:1767319690.459824:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319690.459827:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319690.459835:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0000 x1853163985234176/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319690.459862:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985234048:0@lo:400:kworker.0 00000100:00100000:0.0:1767319690.459917:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319690.459966:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319690.459971:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985234304:0@lo:400:kworker.0 00000100:00100000:0.0:1767319690.459991:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319690.460013:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234304 00000100:00100000:0.0:1767319690.460018:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234304 00000100:00100000:0.0:1767319690.460072:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3127 00000100:00100000:0.0:1767319690.460081:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985234304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319690.460126:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985234304:12345-0@lo:400:kworker.0 Request processed in 46us (132us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319690.460134:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3127 00000100:00080000:0.0:1767319690.460154:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319690.460213:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985234304:0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.460324:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985233792:0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.460353:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319690.460376:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319690.460381:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985234432:0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.460401:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319690.460424:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234432 00000100:00100000:1.0:1767319690.460427:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234432 00000100:00100000:1.0:1767319690.460455:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3128 00000100:00100000:1.0:1767319690.460459:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985234432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.460485:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985234432:12345-0@lo:400:kworker.0 Request processed in 25us (83us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319690.460490:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3128 00000100:00080000:1.0:1767319690.460504:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319690.460518:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985234432:0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.460535:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234048 00000100:00100000:1.0:1767319690.460538:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234048 00000100:00100000:1.0:1767319690.460560:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985233792 00000100:00100000:1.0:1767319690.460562:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985233792 00000100:00100000:1.0:1767319690.460572:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4770 00000100:00100000:1.0:1767319690.460575:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985234048:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.460599:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985234048:12345-0@lo:400:kworker.0 Request processed in 23us (684us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319690.460602:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4770 00000100:00100000:1.0:1767319690.460609:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4771 00000100:00100000:1.0:1767319690.460612:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985233792:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.460629:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985233792:12345-0@lo:400:kworker.0 Request processed in 16us (276us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319690.460632:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4771 00000100:00080000:1.0:1767319690.460639:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319690.460648:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985233792:0@lo:400:kworker.0 00000100:00100000:0.0:1767319690.460708:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985234048:0@lo:400:kworker.0 00000100:00100000:3.0:1767319690.460876:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985233408:0@lo:250:kworker.0 00000400:00000100:3.0:1767319690.460938:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985233408 offset 0 length 520: 4 00000100:00100000:3.0:1767319690.460977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985233536:0@lo:38:kworker.0 00000100:00100000:3.0:1767319690.461011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319690.461043:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985233664:0@lo:38:kworker.0 00000100:00100000:3.0:1767319690.461084:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319690.461121:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985233920:0@lo:38:kworker.0 00000100:00100000:3.0:1767319690.461144:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319690.461202:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985234176:0@lo:38:kworker.0 00000100:00100000:3.0:1767319690.461225:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319690.461266:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985233536 00000100:00100000:3.0:1767319690.461274:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985233536 00000100:00100000:3.0:1767319690.461306:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985233920 00000100:00100000:3.0:1767319690.461310:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985233920 00000100:00100000:3.0:1767319690.461323:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234176 00000100:00100000:3.0:1767319690.461326:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234176 00000100:00100000:3.0:1767319690.461351:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5296 00000100:00100000:3.0:1767319690.461360:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985233536:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319690.461370:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5297 00000100:00100000:2.0:1767319690.461379:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985233920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319690.461386:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767319690.461401:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319690.461443:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985233536:12345-0@lo:38:kworker.0 Request processed in 81us (429us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319690.461453:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5296 00000100:00100000:2.0:1767319690.461461:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985233920:12345-0@lo:38:kworker.0 Request processed in 81us (314us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319690.461467:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5297 00000100:00100000:3.0:1767319690.461482:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5298 00000100:00100000:3.0:1767319690.461492:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985234176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319690.461504:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319690.461547:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985234176:12345-0@lo:38:kworker.0 Request processed in 55us (318us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319690.461554:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5298 00000100:00080000:2.0:1767319690.461560:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.461566:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319690.461571:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985233536:0@lo:38:kworker.0 00000100:00080000:2.0:1767319690.461582:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.461585:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319690.461587:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985233920:0@lo:38:kworker.0 00000100:00080000:2.0:1767319690.461599:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.461601:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319690.461603:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985234176:0@lo:38:kworker.0 00000100:00100000:0.0:1767319690.463576:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985233664 00000100:00100000:0.0:1767319690.463593:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985233664 00000100:00100000:0.0:1767319690.463663:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1814 00000100:00100000:0.0:1767319690.463672:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985233664:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319690.463695:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319690.463809:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985233664:12345-0@lo:38:kworker.0 Request processed in 136us (2722us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319690.463817:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1814 00000100:00080000:2.0:1767319690.464017:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.464026:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319690.464030:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985233664:0@lo:38:kworker.0 00010000:00010000:1.0:1767319691.099764:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319692.123398:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319693.147384:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319694.043686:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985234688:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319694.043707:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985234560:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319694.043757:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319694.043803:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319694.043874:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234688 00000100:00100000:2.0:1767319694.043883:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234688 00000100:00100000:0.0:1767319694.043893:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234560 00000100:00100000:0.0:1767319694.043900:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234560 00000100:00080000:1.0:1767319694.043972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319694.043976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319694.044064:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2304 00000100:00100000:0.0:1767319694.044074:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985234560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319694.044114:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319694.044202:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985234560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 129us (398us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319694.044210:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2304 00000100:00100000:0.0:1767319694.044231:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2305 00000100:00100000:0.0:1767319694.044239:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985234688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319694.044256:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319694.044292:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985234688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 54us (535us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319694.044298:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2305 00000100:00100000:2.0:1767319694.044339:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985234688:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319694.046453:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985234560:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319694.171414:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319695.195770:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319695.532648:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319695.532880:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319695.533253:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920575232 00000100:00100000:1.0:1767319695.533264:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920575232 00000100:00100000:3.0:1767319695.533318:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920575104 00000100:00100000:3.0:1767319695.533324:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920575104 00000100:00100000:1.0:1767319695.533357:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4772 00000100:00080000:2.0:1767319695.533417:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319695.533458:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920575232:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319695.533545:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920575232:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (908us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319695.534786:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5299 00000100:00100000:3.0:1767319695.534796:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920575104:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319695.534818:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319695.534872:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920575104:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 76us (1991us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319695.534880:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5299 00000100:00100000:1.0:1767319695.537861:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4772 00000100:00080000:2.0:1767319695.581230:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.581240:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319695.581243:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.581247:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.581251:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.581259:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4938 00000100:00080000:2.0:1767319695.581263:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.581270:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319695.581303:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357bb80 x1853163985234816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319695.581331:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.581334:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.581336:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.581339:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4938 00000100:00080000:2.0:1767319695.581341:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.581345:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319695.581358:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985234944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319695.581411:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985235072:0@lo:400:kworker.0 00000100:00100000:3.0:1767319695.581456:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985234816:0@lo:38:kworker.0 00000100:00100000:1.0:1767319695.581477:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319695.581588:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319695.581625:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.581630:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.581631:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.581636:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4938 00000100:00080000:2.0:1767319695.581638:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.581642:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319695.581655:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579f80 x1853163985235200/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319695.581679:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.581683:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.581684:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.581688:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4938 00000100:00080000:2.0:1767319695.581690:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.581693:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319695.581702:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578e00 x1853163985235456/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319695.581809:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319695.581815:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985235712:0@lo:400:kworker.0 00000100:00100000:1.0:1767319695.581853:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319695.581878:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985235584:0@lo:400:kworker.0 00000100:00100000:1.0:1767319695.582144:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319695.582193:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319695.582199:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985235328:0@lo:400:kworker.0 00000100:00100000:1.0:1767319695.582227:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235712 00000100:00100000:1.0:1767319695.582233:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235712 00000100:00100000:1.0:1767319695.582270:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235584 00000100:00100000:1.0:1767319695.582273:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235584 00000100:00100000:1.0:1767319695.582292:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3129 00000100:00100000:1.0:1767319695.582299:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985235712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319695.582346:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985235712:12345-0@lo:400:kworker.0 Request processed in 46us (492us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319695.582353:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3129 00000100:00100000:1.0:1767319695.582369:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3130 00000100:00100000:1.0:1767319695.582375:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985235584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319695.582404:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985235584:12345-0@lo:400:kworker.0 Request processed in 29us (506us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319695.582413:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3130 00000100:00100000:1.0:1767319695.582455:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985235712:0@lo:400:kworker.0 00000100:00100000:0.0:1767319695.582827:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235072 00000100:00100000:0.0:1767319695.582832:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235072 00000100:00100000:3.0:1767319695.582868:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985234944:0@lo:38:kworker.0 00000100:00100000:0.0:1767319695.582885:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234816 00000100:00100000:0.0:1767319695.582889:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234816 00000100:00100000:3.0:1767319695.582900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319695.582942:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985235200:0@lo:38:kworker.0 00000100:00100000:3.0:1767319695.582973:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319695.582988:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985235456:0@lo:38:kworker.0 00000100:00100000:3.0:1767319695.583005:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319695.583032:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985234944 00000100:00100000:3.0:1767319695.583036:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985234944 00000100:00100000:3.0:1767319695.583057:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1815 00000100:00100000:3.0:1767319695.583062:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985234944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319695.583079:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319695.583117:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985234944:12345-0@lo:38:kworker.0 Request processed in 53us (216us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319695.583123:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1815 00000100:00100000:3.0:1767319695.583142:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235200 00000100:00100000:3.0:1767319695.583146:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235200 00000100:00100000:0.0:1767319695.583184:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235456 00000100:00100000:0.0:1767319695.583189:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235456 00000100:00100000:1.0:1767319695.584040:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4773 00000100:00100000:1.0:1767319695.584054:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985235072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319695.584115:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985235072:12345-0@lo:400:kworker.0 Request processed in 65us (2639us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319695.584122:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4773 00000100:00100000:1.0:1767319695.584140:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5300 00000100:00100000:1.0:1767319695.584144:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985234816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319695.584158:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319695.584202:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985234816:12345-0@lo:38:kworker.0 Request processed in 56us (2614us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319695.584207:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5300 00000100:00100000:1.0:1767319695.584218:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5301 00000100:00100000:1.0:1767319695.584222:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985235200:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319695.584231:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319695.584368:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5302 00000100:00100000:3.0:1767319695.584372:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985235456:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319695.584380:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319695.584436:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985235456:12345-0@lo:38:kworker.0 Request processed in 62us (1429us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319695.584440:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5302 00000100:00080000:3.0:1767319695.584479:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319695.584482:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319695.584485:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985235456:0@lo:38:kworker.0 00000100:00080000:3.0:1767319695.584490:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319695.584493:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319695.584495:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985234816:0@lo:38:kworker.0 00000100:00080000:3.0:1767319695.584501:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319695.584503:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319695.584505:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985234944:0@lo:38:kworker.0 00000100:00100000:3.0:1767319695.584536:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319695.584559:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985235584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319695.584570:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235328 00000100:00100000:3.0:1767319695.584573:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235328 00000100:00100000:3.0:1767319695.584591:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4774 00000100:00100000:3.0:1767319695.584594:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985235328:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319695.584616:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985235328:12345-0@lo:400:kworker.0 Request processed in 21us (79us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319695.584620:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4774 00000100:00100000:3.0:1767319695.584639:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985235328:0@lo:400:kworker.0 00000100:00080000:3.0:1767319695.584674:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319695.584707:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985235072:0@lo:400:kworker.0 00000100:00100000:1.0:1767319695.584761:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985235200:12345-0@lo:38:kworker.0 Request processed in 537us (1787us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319695.584765:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5301 00000100:00080000:3.0:1767319695.584811:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319695.584815:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319695.584818:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985235200:0@lo:38:kworker.0 00010000:00010000:1.0:1767319696.227195:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319697.244935:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319698.267377:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319699.163321:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319699.163335:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985235840:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319699.163387:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319699.163551:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235840 00000100:00100000:1.0:1767319699.163557:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235840 00000100:00100000:1.0:1767319699.163605:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2306 00000100:00100000:1.0:1767319699.163611:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985235840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319699.163634:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319699.163669:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985235840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319699.163674:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2306 00000100:00080000:0.0:1767319699.163728:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319699.164133:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319699.164147:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985235968:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319699.164212:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319699.164323:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985235840:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319699.164417:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985235968 00000100:00100000:0.0:1767319699.164422:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985235968 00000100:00100000:0.0:1767319699.164502:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2307 00000100:00100000:0.0:1767319699.164509:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985235968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319699.164528:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319699.164573:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985235968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 64us (360us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319699.164579:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2307 00000100:00100000:2.0:1767319699.164689:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985235968:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319699.164813:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319699.291406:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319700.316148:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319700.651388:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319700.651441:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319700.651591:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920575488 00000100:00100000:0.0:1767319700.651602:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920575488 00000100:00100000:0.0:1767319700.651674:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920575360 00000100:00100000:0.0:1767319700.651678:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920575360 00000100:00100000:0.0:1767319700.651694:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4775 00000100:00080000:2.0:1767319700.651725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319700.651758:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920575488:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319700.651843:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920575488:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (464us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319700.656631:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5303 00000100:00100000:1.0:1767319700.656644:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920575360:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319700.656671:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319700.656910:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920575360:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 265us (5466us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319700.656919:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5303 00000100:00100000:0.0:1767319700.662207:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4775 00000100:00080000:2.0:1767319700.699589:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319700.699600:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319700.699603:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319700.699608:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319700.699611:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319700.699619:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4943 00000100:00080000:2.0:1767319700.699624:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319700.699630:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319700.699672:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578380 x1853163985236096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319700.703935:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319700.703945:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319700.703949:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319700.703956:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4943 00000100:00080000:2.0:1767319700.703960:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319700.703968:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319700.704009:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985236224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319700.704065:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319700.704069:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319700.704070:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319700.704074:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4943 00000100:00080000:2.0:1767319700.704076:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319700.704079:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319700.704093:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578e00 x1853163985236480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319700.704110:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1767319700.704117:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319700.704120:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319700.704122:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319700.704125:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4943 00000100:00080000:2.0:1767319700.704127:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319700.704127:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985236352:0@lo:400:kworker.0 00000100:00080000:2.0:1767319700.704130:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319700.704139:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357aa00 x1853163985236736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319700.704187:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985236864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319700.704207:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319700.704215:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319700.704263:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985236992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319700.704295:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236352 00000100:00100000:3.0:1767319700.704304:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236352 00000100:00100000:3.0:1767319700.704361:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4776 00000100:00100000:3.0:1767319700.704381:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985236352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319700.704431:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985236352:12345-0@lo:400:kworker.0 Request processed in 62us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319700.704438:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4776 00000100:00100000:3.0:1767319700.704470:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985236608:0@lo:400:kworker.0 00000100:00100000:3.0:1767319700.704589:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319700.704626:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236608 00000100:00100000:3.0:1767319700.704630:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236608 00000100:00100000:3.0:1767319700.704651:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4777 00000100:00100000:3.0:1767319700.704657:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985236608:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319700.704684:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985236608:12345-0@lo:400:kworker.0 Request processed in 28us (96us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319700.704690:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4777 00000100:00100000:3.0:1767319700.704719:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985236608:0@lo:400:kworker.0 00000100:00080000:3.0:1767319700.704744:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319700.705129:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236864 00000100:00100000:1.0:1767319700.705134:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236864 00000100:00100000:1.0:1767319700.705199:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3131 00000100:00100000:1.0:1767319700.705207:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985236864:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319700.705245:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985236864:12345-0@lo:400:kworker.0 Request processed in 40us (1028us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319700.705251:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3131 00000100:00100000:2.0:1767319700.705401:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985236864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319700.706350:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319700.706404:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985236352:0@lo:400:kworker.0 00000100:00100000:2.0:1767319700.706477:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236992 00000100:00100000:2.0:1767319700.706482:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236992 00000100:00100000:2.0:1767319700.706529:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3132 00000100:00100000:2.0:1767319700.706537:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985236992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319700.706591:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985236992:12345-0@lo:400:kworker.0 Request processed in 54us (243us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319700.706761:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985236992:0@lo:400:kworker.0 00000100:00100000:2.0:1767319700.706797:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3132 00000100:00100000:0.0:1767319700.706935:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985236096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319700.706971:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319700.707046:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985236224:0@lo:38:kworker.0 00000100:00100000:0.0:1767319700.707070:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319700.707100:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985236480:0@lo:38:kworker.0 00000100:00100000:0.0:1767319700.707120:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319700.707125:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236224 00000100:00100000:3.0:1767319700.707128:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236224 00000100:00100000:3.0:1767319700.707209:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1816 00000100:00100000:3.0:1767319700.707214:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985236224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319700.707232:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319700.707262:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985236224:12345-0@lo:38:kworker.0 Request processed in 46us (191us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319700.707268:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1816 00000100:00100000:0.0:1767319700.707551:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985236736:0@lo:38:kworker.0 00000100:00100000:0.0:1767319700.707574:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319700.707604:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236096 00000100:00100000:1.0:1767319700.707609:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236096 00000100:00100000:1.0:1767319700.707638:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236480 00000100:00100000:1.0:1767319700.707641:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236480 00000100:00100000:1.0:1767319700.707659:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5304 00000100:00100000:1.0:1767319700.707663:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985236096:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319700.707673:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5305 00010000:00020000:1.0:1767319700.707674:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319700.707677:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985236480:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319700.707685:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319700.707703:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985236096:12345-0@lo:38:kworker.0 Request processed in 38us (731us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319700.707706:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985236480:12345-0@lo:38:kworker.0 Request processed in 28us (585us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319700.707709:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5304 00000100:00100000:2.0:1767319700.707712:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5305 00000100:00080000:0.0:1767319700.709272:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319700.709284:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319700.709288:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985236096:0@lo:38:kworker.0 00000100:00080000:0.0:1767319700.709300:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319700.709304:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319700.709307:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985236224:0@lo:38:kworker.0 00000100:00080000:0.0:1767319700.709315:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319700.709318:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319700.709321:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985236480:0@lo:38:kworker.0 00000100:00100000:2.0:1767319700.709814:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985236736 00000100:00100000:2.0:1767319700.709823:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985236736 00000100:00100000:2.0:1767319700.709878:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5306 00000100:00100000:2.0:1767319700.709886:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985236736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319700.709905:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319700.709978:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985236736:12345-0@lo:38:kworker.0 Request processed in 91us (2402us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319700.709985:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5306 00000100:00080000:0.0:1767319700.710206:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319700.710211:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319700.710214:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985236736:0@lo:38:kworker.0 00010000:00010000:2.0:1767319701.340783:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319702.367266:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319703.387938:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319704.283838:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985237248:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319704.283935:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319704.284107:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985237248 00000100:00100000:0.0:1767319704.284118:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985237248 00000100:00100000:0.0:1767319704.284215:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2308 00000100:00100000:0.0:1767319704.284227:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985237248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319704.284266:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319704.284319:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985237248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (384us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319704.284327:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2308 00000100:00080000:2.0:1767319704.284404:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319704.284459:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985237248:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319704.284630:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985237120:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319704.284669:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319704.285190:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985237120 00000100:00100000:0.0:1767319704.285196:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985237120 00000100:00100000:0.0:1767319704.285248:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2309 00000100:00100000:0.0:1767319704.285257:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985237120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319704.285279:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319704.285332:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985237120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (662us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319704.285340:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2309 00000100:00080000:2.0:1767319704.285446:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319704.285982:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985237120:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319704.417713:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319705.439145:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319705.769643:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319705.769732:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920575616 00000100:00100000:1.0:1767319705.769746:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920575616 00000100:00100000:1.0:1767319705.769809:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5307 00000100:00100000:1.0:1767319705.769818:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920575616:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319705.769848:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319705.769943:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920575616:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 125us (314us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319705.769952:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5307 00000100:00100000:2.0:1767319705.789572:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319705.789664:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920575744 00000100:00100000:3.0:1767319705.789675:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920575744 00000100:00100000:3.0:1767319705.789747:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4778 00000100:00100000:3.0:1767319705.789758:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920575744:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:2.0:1767319705.789959:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319705.790007:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920575744:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 250us (444us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319705.790016:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4778 00000100:00080000:2.0:1767319705.819498:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.819509:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319705.819512:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.819516:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.819521:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.819528:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.819533:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.819539:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319705.819583:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21880 x1853163985237376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319705.819620:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.819623:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.819626:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.819630:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.819633:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.819637:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319705.819650:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985237504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319705.819696:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.819700:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.819702:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.819705:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.819708:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.819711:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319705.819727:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20a80 x1853163985237760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319705.819755:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.819759:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.819761:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.819764:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.819777:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.819780:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319705.819793:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22d80 x1853163985238016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319705.819855:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985238144:0@lo:400:kworker.0 00000100:00100000:2.0:1767319705.819921:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319705.819985:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319705.819991:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985237888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319705.820015:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319705.820324:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238144 00000100:00100000:2.0:1767319705.820331:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238144 00000100:00100000:2.0:1767319705.820394:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3133 00000100:00100000:2.0:1767319705.820404:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985238144:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319705.820457:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985238144:12345-0@lo:400:kworker.0 Request processed in 55us (535us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319705.820464:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3133 00000100:00100000:2.0:1767319705.820504:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985238144:0@lo:400:kworker.0 00000100:00100000:3.0:1767319705.820624:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985237888 00000100:00100000:3.0:1767319705.820629:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985237888 00000100:00100000:3.0:1767319705.820677:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4779 00000100:00100000:3.0:1767319705.820683:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985237888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319705.820717:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985237888:12345-0@lo:400:kworker.0 Request processed in 35us (701us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319705.820722:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4779 00000100:00100000:3.0:1767319705.820743:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985237632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319705.820776:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319705.820798:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319705.820803:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985238272:0@lo:400:kworker.0 00000100:00100000:3.0:1767319705.820836:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319705.820862:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985237632 00000100:00100000:3.0:1767319705.820865:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985237632 00000100:00100000:3.0:1767319705.820884:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4780 00000100:00100000:3.0:1767319705.820889:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985237632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319705.820918:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985237632:12345-0@lo:400:kworker.0 Request processed in 29us (152us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319705.820923:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4780 00000100:00100000:3.0:1767319705.820944:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985237632:0@lo:400:kworker.0 00000100:00100000:0.0:1767319705.821648:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985237376:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.821686:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.821719:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985237504:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.821742:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.821780:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985237760:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.821801:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.821832:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985238016:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.821942:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.821976:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985237376 00000100:00100000:0.0:1767319705.821981:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985237376 00000100:00100000:0.0:1767319705.822001:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985237760 00000100:00100000:0.0:1767319705.822004:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985237760 00000100:00100000:0.0:1767319705.822013:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238016 00000100:00100000:0.0:1767319705.822016:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238016 00000100:00100000:0.0:1767319705.822026:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5308 00000100:00100000:0.0:1767319705.822032:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985237376:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319705.822222:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5309 00000100:00100000:1.0:1767319705.822228:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985237760:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319705.822240:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319705.822286:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985237760:12345-0@lo:38:kworker.0 Request processed in 55us (482us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319705.822292:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5309 00000100:00100000:1.0:1767319705.822309:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5310 00000100:00100000:1.0:1767319705.822313:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985238016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319705.822322:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319705.822344:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985238016:12345-0@lo:38:kworker.0 Request processed in 29us (402us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319705.822349:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5310 00000100:00080000:1.0:1767319705.822472:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319705.822478:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319705.822481:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985237760:0@lo:38:kworker.0 00000100:00080000:1.0:1767319705.822490:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319705.822493:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319705.822496:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985238016:0@lo:38:kworker.0 00000100:00100000:2.0:1767319705.822578:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238272 00000100:00100000:2.0:1767319705.822582:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238272 00000100:00100000:2.0:1767319705.822621:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3134 00000100:00100000:2.0:1767319705.822628:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985238272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319705.822669:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985238272:12345-0@lo:400:kworker.0 Request processed in 42us (1832us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319705.822675:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3134 00000100:00080000:2.0:1767319705.822690:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319705.822711:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985237888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319705.822736:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985238272:0@lo:400:kworker.0 00010000:00020000:0.0:1767319705.822809:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319705.823023:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985237504 00000100:00100000:3.0:1767319705.823028:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985237504 00000100:00100000:3.0:1767319705.823210:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1817 00000100:00100000:3.0:1767319705.823217:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985237504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319705.823229:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319705.823985:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985237376:12345-0@lo:38:kworker.0 Request processed in 1948us (2294us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319705.823994:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5308 00000100:00100000:3.0:1767319705.827364:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985237504:12345-0@lo:38:kworker.0 Request processed in 4134us (5609us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319705.827380:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1817 00000100:00080000:2.0:1767319705.827499:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319705.827505:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319705.827509:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985237376:0@lo:38:kworker.0 00000100:00080000:2.0:1767319705.827526:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319705.827530:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319705.827532:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985237504:0@lo:38:kworker.0 00010000:00010000:1.0:1767319706.459903:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319707.485331:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319708.507400:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319709.403488:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985238400:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319709.403583:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985238528:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319709.403834:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319709.404239:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319709.410850:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238400 00000100:00100000:1.0:1767319709.410867:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238400 00000100:00100000:1.0:1767319709.410996:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238528 00000100:00100000:1.0:1767319709.411000:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238528 00000100:00100000:3.0:1767319709.411059:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2310 00000100:00100000:3.0:1767319709.411071:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985238400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319709.411104:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319709.411187:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985238400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 119us (7353us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319709.411197:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2310 00000100:00100000:3.0:1767319709.411250:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985238400:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319709.413494:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319709.413547:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319709.415308:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2311 00000100:00100000:1.0:1767319709.415325:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985238528:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319709.415364:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319709.415450:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985238528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 130us (11209us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319709.415458:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2311 00000100:00100000:1.0:1767319709.415570:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985238528:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319709.531364:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319710.555431:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319710.889964:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319710.890056:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319710.890248:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920576000 00000100:00100000:3.0:1767319710.890258:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920576000 00000100:00100000:3.0:1767319710.890334:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920575872 00000100:00100000:3.0:1767319710.890337:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920575872 00000100:00100000:3.0:1767319710.890358:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4781 00000100:00100000:3.0:1767319710.890368:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920576000:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319710.890451:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920576000:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (496us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319710.890458:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4781 00000100:00080000:2.0:1767319710.890864:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319710.890999:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5311 00000100:00100000:3.0:1767319710.891006:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920575872:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319710.891027:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319710.891069:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920575872:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 62us (1012us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319710.892190:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5311 00000100:00080000:1.0:1767319710.939381:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319710.939392:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319710.939397:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319710.939401:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319710.939405:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319710.939413:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4954 00000100:00080000:1.0:1767319710.939418:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319710.939425:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319710.939461:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985238656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319710.939494:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319710.939498:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319710.939500:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319710.939503:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4954 00000100:00080000:1.0:1767319710.939506:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319710.939509:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319710.939524:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2d80 x1853163985238784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319710.939572:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319710.939576:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319710.939578:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319710.939582:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4954 00000100:00080000:1.0:1767319710.939584:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319710.939587:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319710.939599:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3b80 x1853163985239040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319710.939632:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319710.939635:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319710.939637:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319710.939640:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4954 00000100:00080000:1.0:1767319710.939642:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319710.939645:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319710.939658:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1c00 x1853163985239296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319710.939708:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985239168:0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.939769:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.939832:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319710.939838:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985239424:0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.939866:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.939894:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985239424 00000100:00100000:1.0:1767319710.939901:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985239424 00000100:00100000:1.0:1767319710.939946:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3135 00000100:00100000:1.0:1767319710.939955:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985239424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.940004:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985239424:12345-0@lo:400:kworker.0 Request processed in 50us (136us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319710.940011:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3135 00000100:00100000:1.0:1767319710.940051:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985239424:0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.940073:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985239168 00000100:00100000:1.0:1767319710.940076:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985239168 00000100:00100000:1.0:1767319710.940111:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4782 00000100:00100000:1.0:1767319710.940116:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985239168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.940148:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985239168:12345-0@lo:400:kworker.0 Request processed in 32us (379us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319710.940154:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4782 00000100:00100000:1.0:1767319710.940210:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985239168:0@lo:400:kworker.0 00000100:00100000:0.0:1767319710.940315:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985239552:0@lo:400:kworker.0 00000100:00100000:0.0:1767319710.940344:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319710.940371:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319710.940376:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985238912:0@lo:400:kworker.0 00000100:00100000:0.0:1767319710.940397:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319710.940426:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238912 00000100:00100000:0.0:1767319710.940430:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238912 00000100:00100000:0.0:1767319710.940457:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4783 00000100:00100000:0.0:1767319710.940464:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985238912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319710.940498:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985238912:12345-0@lo:400:kworker.0 Request processed in 35us (100us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319710.940504:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4783 00000100:00100000:0.0:1767319710.940528:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985238912:0@lo:400:kworker.0 00000100:00100000:2.0:1767319710.940602:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985238656:0@lo:38:kworker.0 00000100:00100000:2.0:1767319710.940630:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319710.940655:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985238784:0@lo:38:kworker.0 00000100:00100000:2.0:1767319710.940685:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319710.940709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985239040:0@lo:38:kworker.0 00000100:00100000:2.0:1767319710.940728:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319710.940741:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985239296:0@lo:38:kworker.0 00000100:00100000:2.0:1767319710.940758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319710.940787:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238784 00000100:00100000:2.0:1767319710.940791:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238784 00000100:00100000:2.0:1767319710.940813:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1818 00000100:00100000:2.0:1767319710.940817:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985238784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319710.940835:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319710.940865:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985238784:12345-0@lo:38:kworker.0 Request processed in 46us (179us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319710.940871:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1818 00000100:00100000:2.0:1767319710.940893:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985239552 00000100:00100000:2.0:1767319710.940896:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985239552 00000100:00100000:2.0:1767319710.940920:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3136 00000100:00100000:2.0:1767319710.940927:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985239552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319710.940961:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985239552:12345-0@lo:400:kworker.0 Request processed in 35us (617us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319710.940967:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3136 00000100:00100000:2.0:1767319710.940994:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985239552:0@lo:400:kworker.0 00000100:00080000:2.0:1767319710.941015:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319710.941020:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319710.941023:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985238784:0@lo:38:kworker.0 00000100:00100000:2.0:1767319710.941039:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985238656 00000100:00100000:2.0:1767319710.941042:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985238656 00000100:00100000:2.0:1767319710.941060:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985239040 00000100:00100000:2.0:1767319710.941062:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985239040 00000100:00100000:2.0:1767319710.941075:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985239296 00000100:00100000:2.0:1767319710.941077:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985239296 00000100:00100000:2.0:1767319710.941090:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5312 00000100:00100000:2.0:1767319710.941094:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985238656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319710.941104:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319710.941130:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985238656:12345-0@lo:38:kworker.0 Request processed in 35us (499us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319710.941136:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5312 00000100:00100000:2.0:1767319710.941148:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5313 00000100:00100000:2.0:1767319710.941151:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985239040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319710.941159:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319710.941203:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985239040:12345-0@lo:38:kworker.0 Request processed in 50us (473us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319710.941208:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5313 00000100:00100000:2.0:1767319710.941219:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5314 00000100:00100000:2.0:1767319710.941223:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985239296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319710.941231:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319710.941252:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985239296:12345-0@lo:38:kworker.0 Request processed in 27us (493us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319710.941257:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5314 00000100:00080000:2.0:1767319710.941289:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319710.941293:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319710.941296:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985238656:0@lo:38:kworker.0 00000100:00080000:2.0:1767319710.941303:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319710.941306:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319710.941309:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985239040:0@lo:38:kworker.0 00000100:00080000:2.0:1767319710.941316:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319710.941319:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319710.941322:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985239296:0@lo:38:kworker.0 00000100:00080000:2.0:1767319710.941344:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319711.579750:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319712.603388:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319713.627651:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319714.523689:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985239680:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319714.523756:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319714.523863:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985239680 00000100:00100000:1.0:1767319714.523871:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985239680 00000100:00100000:1.0:1767319714.523928:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2312 00000100:00100000:1.0:1767319714.523938:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985239680:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319714.523965:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767319714.524061:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319714.524277:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319714.524289:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985239808:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319714.524319:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319714.524626:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985239808 00000100:00100000:0.0:1767319714.524631:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985239808 00000100:00100000:0.0:1767319714.524675:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2313 00000100:00100000:0.0:1767319714.524683:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985239808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319714.524700:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319714.524737:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985239808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (417us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319714.524744:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2313 00000100:00100000:0.0:1767319714.524787:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985239808:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319714.524986:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319714.525294:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985239680:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1355us (1536us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319714.525301:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2312 00000100:00100000:3.0:1767319714.525719:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985239680:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319714.653214:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319715.035261:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319690/real 1767319690] req@ffff8ce188f04e00 x1853163985233408/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319715 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319715.035286:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188f04e00 x1853163985233408/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319715 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319715.035306:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319715.035327:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319715.035330:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985233408:0@lo:250:kworker.0 00000100:00080000:0.0:1767319715.035399:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.035403:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319715.035406:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.035409:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4938 00000100:00080000:0.0:1767319715.035412:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.035417:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319715.035443:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3b80 x1853163985239936/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319715.035466:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.035468:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.035470:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.035472:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.035473:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.035475:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319715.035489:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c700 x1853163985240064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319715.035497:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.035500:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.035501:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.035504:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.035506:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.035509:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319715.035520:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18f480 x1853163985240192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319715.035566:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.035570:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.035572:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.035576:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.035578:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.035581:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319715.035592:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18d880 x1853163985240448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319715.035628:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.035632:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.035633:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.035637:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.035639:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.035643:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319715.035652:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c380 x1853163985240704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319715.035692:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985240576:0@lo:400:kworker.0 00000100:00100000:0.0:1767319715.035742:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319715.035787:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319715.035791:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985240832:0@lo:400:kworker.0 00000100:00100000:0.0:1767319715.035808:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319715.035829:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240576 00000100:00100000:0.0:1767319715.035834:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240576 00000100:00100000:0.0:1767319715.035877:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4784 00000100:00100000:0.0:1767319715.035885:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985240576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319715.035932:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985240576:12345-0@lo:400:kworker.0 Request processed in 48us (189us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319715.035940:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4784 00000100:00080000:0.0:1767319715.035959:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319715.035978:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985240576:0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.036072:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985239936:0@lo:250:kworker.0 00000400:00000100:2.0:1767319715.036090:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985239936 offset 0 length 520: 4 00000100:00100000:2.0:1767319715.036103:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985240064:0@lo:38:kworker.0 00000100:00100000:2.0:1767319715.036115:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319715.036132:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985240192:0@lo:38:kworker.0 00000100:00100000:2.0:1767319715.036146:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319715.036156:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985240448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319715.036186:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319715.036195:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985240704:0@lo:38:kworker.0 00000100:00100000:2.0:1767319715.036206:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319715.036226:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240192 00000100:00100000:2.0:1767319715.036229:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240192 00000100:00100000:2.0:1767319715.036244:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1819 00000100:00100000:2.0:1767319715.036247:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985240192:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319715.036261:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319715.036281:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985240192:12345-0@lo:38:kworker.0 Request processed in 33us (134us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319715.036286:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1819 00000100:00100000:2.0:1767319715.036298:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240064 00000100:00100000:2.0:1767319715.036300:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240064 00000100:00100000:2.0:1767319715.036311:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240448 00000100:00100000:2.0:1767319715.036312:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240448 00000100:00100000:2.0:1767319715.036318:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240704 00000100:00100000:2.0:1767319715.036320:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240704 00000100:00100000:2.0:1767319715.036327:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5315 00000100:00100000:2.0:1767319715.036329:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985240064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319715.036336:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319715.036349:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985240064:12345-0@lo:38:kworker.0 Request processed in 19us (233us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319715.036353:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5315 00000100:00100000:2.0:1767319715.036358:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5316 00000100:00100000:2.0:1767319715.036360:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985240448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319715.036365:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319715.036375:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985240448:12345-0@lo:38:kworker.0 Request processed in 14us (189us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319715.036378:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5316 00000100:00100000:2.0:1767319715.036383:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5317 00000100:00100000:2.0:1767319715.036385:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985240704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319715.036390:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319715.036400:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985240704:12345-0@lo:38:kworker.0 Request processed in 14us (193us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319715.036403:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5317 00000100:00080000:2.0:1767319715.036426:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319715.036429:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319715.036431:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985240064:0@lo:38:kworker.0 00000100:00080000:2.0:1767319715.036435:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319715.036437:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319715.036439:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985240192:0@lo:38:kworker.0 00000100:00080000:2.0:1767319715.036443:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319715.036445:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319715.036446:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985240448:0@lo:38:kworker.0 00000100:00080000:2.0:1767319715.036450:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319715.036451:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319715.036453:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985240704:0@lo:38:kworker.0 00000100:00100000:3.0:1767319715.036698:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985240960:0@lo:400:kworker.0 00000100:00100000:3.0:1767319715.036730:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319715.036759:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319715.036765:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985240320:0@lo:400:kworker.0 00000100:00100000:3.0:1767319715.036784:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319715.036807:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240320 00000100:00100000:3.0:1767319715.036810:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240320 00000100:00100000:3.0:1767319715.036839:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4785 00000100:00100000:3.0:1767319715.036843:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985240320:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319715.036870:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985240320:12345-0@lo:400:kworker.0 Request processed in 26us (85us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319715.036875:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4785 00000100:00080000:3.0:1767319715.036892:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319715.036909:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985240320:0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.037000:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240832 00000100:00100000:2.0:1767319715.037003:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240832 00000100:00100000:2.0:1767319715.037028:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985240960 00000100:00100000:2.0:1767319715.037030:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985240960 00000100:00100000:2.0:1767319715.037040:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3137 00000100:00100000:2.0:1767319715.037044:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985240832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.037069:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985240832:12345-0@lo:400:kworker.0 Request processed in 26us (1260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319715.037073:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3137 00000100:00100000:2.0:1767319715.037080:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3138 00000100:00100000:2.0:1767319715.037084:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985240960:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.037099:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985240960:12345-0@lo:400:kworker.0 Request processed in 15us (369us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319715.037102:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3138 00000100:00100000:2.0:1767319715.037117:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985240832:0@lo:400:kworker.0 00000100:00100000:3.0:1767319715.037320:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985240960:0@lo:400:kworker.0 00010000:00010000:1.0:1767319715.675397:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319716.013300:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767319716.013340:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920576128 offset 0 length 520: 4 00000100:00100000:0.0:1767319716.013372:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319716.013409:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920576384 00000100:00100000:2.0:1767319716.013421:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920576384 00000100:00100000:3.0:1767319716.013498:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920576256 00000100:00100000:3.0:1767319716.013503:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920576256 00000100:00100000:3.0:1767319716.013549:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5318 00000100:00100000:3.0:1767319716.013559:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920576256:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319716.013591:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319716.013711:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4786 00000100:00100000:2.0:1767319716.013720:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920576384:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319716.013754:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920576256:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 194us (379us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319716.013762:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920576384:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 42us (470us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319716.013769:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5318 00000100:00080000:1.0:1767319716.013788:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319716.013849:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4786 00010000:00010000:2.0:1767319716.699532:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319717.725099:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319718.747524:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319719.643534:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319719.643560:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985241088:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319719.643636:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319719.643786:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241088 00000100:00100000:1.0:1767319719.643794:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241088 00000100:00100000:1.0:1767319719.643865:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2314 00000100:00100000:1.0:1767319719.643874:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985241088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319719.643903:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319719.643951:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985241088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (314us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319719.643959:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2314 00000100:00080000:3.0:1767319719.644218:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319719.644260:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985241088:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319719.644381:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985241216:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319719.644415:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319719.644501:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241216 00000100:00100000:1.0:1767319719.644505:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241216 00000100:00100000:1.0:1767319719.644563:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2315 00000100:00100000:1.0:1767319719.644570:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985241216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319719.644585:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319719.644622:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985241216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319719.644630:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2315 00000100:00100000:2.0:1767319719.644647:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985241216:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319719.644689:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319719.773005:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319720.155351:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.155363:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319720.155366:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.155370:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.155375:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.155383:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4963 00000100:00080000:1.0:1767319720.155387:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.155393:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319720.155433:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0a80 x1853163985241344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319720.155452:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.155455:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.155457:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.155459:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4963 00000100:00080000:1.0:1767319720.155462:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.155465:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319720.155486:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1f80 x1853163985241472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319720.155542:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.155546:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.155548:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.155551:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4963 00000100:00080000:1.0:1767319720.155553:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.155556:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319720.155567:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2680 x1853163985241728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319720.155597:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.155601:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.155603:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.155606:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4963 00000100:00080000:1.0:1767319720.155608:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.155611:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319720.155622:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985241984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319720.155661:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985242112:0@lo:400:kworker.0 00000100:00100000:1.0:1767319720.155719:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319720.155789:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319720.155796:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985241856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319720.155822:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319720.156443:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985242112 00000100:00100000:3.0:1767319720.156451:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985242112 00000100:00100000:3.0:1767319720.156523:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3139 00000100:00100000:3.0:1767319720.156533:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985242112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319720.156592:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985242112:12345-0@lo:400:kworker.0 Request processed in 60us (871us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319720.156598:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3139 00000100:00100000:3.0:1767319720.156623:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241856 00000100:00100000:3.0:1767319720.156626:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241856 00000100:00100000:3.0:1767319720.156660:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4787 00000100:00100000:3.0:1767319720.156666:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985241856:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319720.156695:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985241856:12345-0@lo:400:kworker.0 Request processed in 30us (873us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319720.156701:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4787 00000100:00080000:3.0:1767319720.156717:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319720.156729:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985241600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319720.156754:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319720.156792:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319720.156798:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985242240:0@lo:400:kworker.0 00000100:00100000:3.0:1767319720.156820:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319720.156842:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241600 00000100:00100000:3.0:1767319720.156845:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241600 00000100:00100000:3.0:1767319720.156868:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4788 00000100:00100000:3.0:1767319720.156873:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985241600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319720.156903:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985241600:12345-0@lo:400:kworker.0 Request processed in 29us (148us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319720.156912:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4788 00000100:00080000:3.0:1767319720.156925:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319720.156951:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985241600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319720.157620:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985242240 00000100:00100000:3.0:1767319720.157625:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985242240 00000100:00100000:3.0:1767319720.157652:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3140 00000100:00100000:3.0:1767319720.157659:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985242240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319720.157686:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985241856:0@lo:400:kworker.0 00000100:00100000:2.0:1767319720.157698:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985242112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319720.157700:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985242240:12345-0@lo:400:kworker.0 Request processed in 42us (879us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319720.157705:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3140 00000100:00100000:3.0:1767319720.157728:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985242240:0@lo:400:kworker.0 00000100:00100000:0.0:1767319720.158284:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985241344:0@lo:38:kworker.0 00000100:00100000:0.0:1767319720.158324:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319720.158372:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985241472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319720.158399:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319720.158492:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985241728:0@lo:38:kworker.0 00000100:00100000:0.0:1767319720.158523:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319720.158548:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985241984:0@lo:38:kworker.0 00000100:00100000:0.0:1767319720.158567:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319720.158597:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241344 00000100:00100000:0.0:1767319720.158601:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241344 00000100:00100000:0.0:1767319720.158620:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241728 00000100:00100000:0.0:1767319720.158622:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241728 00000100:00100000:0.0:1767319720.158635:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241984 00000100:00100000:0.0:1767319720.158637:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241984 00000100:00100000:0.0:1767319720.158651:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5319 00000100:00100000:0.0:1767319720.158656:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985241344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319720.158675:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319720.158705:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985241344:12345-0@lo:38:kworker.0 Request processed in 48us (380us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319720.158711:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5319 00000100:00100000:0.0:1767319720.158723:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5320 00000100:00100000:0.0:1767319720.158726:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985241728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319720.158734:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319720.158753:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985241728:12345-0@lo:38:kworker.0 Request processed in 26us (230us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319720.158758:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5320 00000100:00100000:0.0:1767319720.158767:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5321 00000100:00100000:0.0:1767319720.158782:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985241984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319720.158790:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319720.158809:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985241984:12345-0@lo:38:kworker.0 Request processed in 38us (241us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319720.158813:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5321 00000100:00080000:0.0:1767319720.158848:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319720.158854:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319720.158858:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985241344:0@lo:38:kworker.0 00000100:00080000:0.0:1767319720.158866:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319720.158869:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319720.158871:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985241728:0@lo:38:kworker.0 00000100:00080000:0.0:1767319720.158878:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319720.158881:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319720.158884:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985241984:0@lo:38:kworker.0 00000100:00100000:2.0:1767319720.164154:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985241472 00000100:00100000:2.0:1767319720.164186:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985241472 00000100:00100000:0.0:1767319720.171225:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1820 00000100:00100000:0.0:1767319720.171237:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985241472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319720.171260:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319720.171354:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985241472:12345-0@lo:38:kworker.0 Request processed in 117us (12953us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319720.171361:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1820 00000100:00080000:1.0:1767319720.172542:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319720.172551:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319720.172556:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985241472:0@lo:38:kworker.0 00010000:00010000:1.0:1767319720.795398:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319721.140410:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319721.140620:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319721.145617:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920576512 00000100:00100000:3.0:1767319721.145635:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920576512 00000100:00100000:3.0:1767319721.145724:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920576640 00000100:00100000:3.0:1767319721.145729:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920576640 00000100:00080000:1.0:1767319721.145803:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319721.151109:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4789 00000100:00100000:2.0:1767319721.151137:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920576640:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319721.151346:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920576640:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 223us (10725us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319721.151356:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4789 00000100:00100000:2.0:1767319721.151381:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5322 00000100:00100000:2.0:1767319721.151385:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920576512:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319721.151416:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319721.151462:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920576512:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 65us (11054us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319721.151469:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5322 00010000:00010000:1.0:1767319721.820665:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319722.846109:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319723.874151:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319724.763476:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985242368:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319724.763550:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319724.763686:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985242368 00000100:00100000:1.0:1767319724.763694:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985242368 00000100:00100000:1.0:1767319724.763772:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2316 00000100:00100000:1.0:1767319724.763782:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985242368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319724.763812:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319724.763861:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985242368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (311us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319724.763869:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2316 00000100:00100000:1.0:1767319724.763916:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985242368:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319724.764139:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319724.764354:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319724.764366:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985242496:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319724.764398:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319724.764480:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985242496 00000100:00100000:0.0:1767319724.764486:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985242496 00000100:00100000:0.0:1767319724.764528:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2317 00000100:00100000:0.0:1767319724.764536:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985242496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319724.764553:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319724.764592:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985242496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (192us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319724.764598:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2317 00000100:00100000:0.0:1767319724.764630:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985242496:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319724.765078:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319724.892689:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319725.277503:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319725.277515:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319725.277518:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319725.277523:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319725.277527:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319725.277536:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4968 00000100:00080000:0.0:1767319725.277541:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319725.277548:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319725.277586:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ed80 x1853163985242624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319725.277606:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319725.277610:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319725.277611:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319725.277615:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4968 00000100:00080000:0.0:1767319725.277618:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319725.277621:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319725.277635:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c000 x1853163985242752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319725.277644:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985242624:0@lo:38:kworker.0 00000100:00080000:0.0:1767319725.277679:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319725.277682:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319725.277684:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319725.277687:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4968 00000100:00080000:0.0:1767319725.277689:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319725.277693:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319725.277703:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18d880 x1853163985243008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319725.277706:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319725.277726:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319725.277729:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319725.277731:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319725.277734:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4968 00000100:00080000:0.0:1767319725.277736:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319725.277738:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319725.277747:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c700 x1853163985243264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319725.277776:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319725.277785:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985243136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319725.277786:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319725.277793:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985242880:0@lo:400:kworker.0 00000100:00100000:0.0:1767319725.277815:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319725.277893:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985242752:0@lo:38:kworker.0 00000100:00100000:1.0:1767319725.277922:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319725.279049:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985242752 00000100:00100000:2.0:1767319725.279059:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985242752 00000100:00100000:3.0:1767319725.279091:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985242624 00000100:00100000:3.0:1767319725.279094:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985242624 00000100:00100000:3.0:1767319725.279125:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243136 00000100:00100000:3.0:1767319725.279128:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243136 00000100:00100000:3.0:1767319725.279182:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4790 00000100:00100000:3.0:1767319725.279196:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985243136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319725.279236:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1821 00000100:00100000:2.0:1767319725.279241:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190db8850 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985242752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319725.279302:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319725.279343:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190db8850 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985242752:12345-0@lo:38:kworker.0 Request processed in 101us (1420us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319725.279349:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1821 00000100:00100000:2.0:1767319725.279532:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5323 00000100:00100000:2.0:1767319725.279538:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985242624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319725.279549:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319725.279585:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985243136:12345-0@lo:400:kworker.0 Request processed in 396us (1769us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319725.279591:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4790 00000100:00080000:3.0:1767319725.279612:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319725.279647:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319725.279698:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985242624:12345-0@lo:38:kworker.0 Request processed in 159us (1992us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319725.279704:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5323 00000100:00100000:2.0:1767319725.279713:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985242880 00000100:00100000:2.0:1767319725.279716:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985242880 00000100:00100000:2.0:1767319725.279737:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4791 00000100:00100000:2.0:1767319725.279742:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985242880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319725.279767:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985242880:12345-0@lo:400:kworker.0 Request processed in 24us (119us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319725.279772:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4791 00000100:00080000:2.0:1767319725.279793:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319725.282031:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319725.282036:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985243136:0@lo:400:kworker.0 00000100:00080000:1.0:1767319725.282038:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319725.282041:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985242752:0@lo:38:kworker.0 00000100:00100000:1.0:1767319725.282052:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985243008:0@lo:38:kworker.0 00000100:00100000:1.0:1767319725.282112:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319725.282151:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985243264:0@lo:38:kworker.0 00000100:00100000:1.0:1767319725.282200:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319725.282226:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319725.282230:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319725.282233:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985242624:0@lo:38:kworker.0 00000100:00100000:0.0:1767319725.282318:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243008 00000100:00100000:0.0:1767319725.282325:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243008 00000100:00100000:0.0:1767319725.282377:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243264 00000100:00100000:0.0:1767319725.282380:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243264 00000100:00100000:0.0:1767319725.282401:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5324 00000100:00100000:0.0:1767319725.282420:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985243008:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319725.282441:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319725.282543:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985243008:12345-0@lo:38:kworker.0 Request processed in 133us (430us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319725.282551:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5324 00000100:00100000:0.0:1767319725.282568:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5325 00000100:00100000:0.0:1767319725.282572:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985243264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319725.282582:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319725.282607:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985243264:12345-0@lo:38:kworker.0 Request processed in 33us (406us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319725.282612:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5325 00000100:00080000:1.0:1767319725.284951:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319725.284961:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319725.284964:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985243008:0@lo:38:kworker.0 00000100:00080000:1.0:1767319725.284972:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319725.284974:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319725.284976:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985243264:0@lo:38:kworker.0 00000100:00100000:3.0:1767319725.287253:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985242880:0@lo:400:kworker.0 00010000:00010000:2.0:1767319725.917069:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319726.258071:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319726.258209:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319726.258291:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920576768 00000100:00100000:0.0:1767319726.258301:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920576768 00000100:00100000:1.0:1767319726.258309:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920576896 00000100:00100000:1.0:1767319726.258315:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920576896 00000100:00100000:0.0:1767319726.258365:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5326 00000100:00100000:0.0:1767319726.258373:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920576768:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319726.258408:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319726.258482:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319726.258483:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920576768:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 108us (422us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319726.272760:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4792 00000100:00100000:2.0:1767319726.272787:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920576896:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319726.272900:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920576896:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 126us (14690us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319726.272909:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4792 00000100:00100000:0.0:1767319726.275490:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5326 00010000:00010000:1.0:1767319726.939388:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319727.963337:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319728.989737:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319729.883398:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985243392:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319729.883470:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319729.883887:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243392 00000100:00100000:3.0:1767319729.883895:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243392 00000100:00100000:3.0:1767319729.883955:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2318 00000100:00100000:3.0:1767319729.883964:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985243392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319729.883993:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319729.884037:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985243392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (566us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319729.884044:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2318 00000100:00080000:3.0:1767319729.884066:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319729.884334:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319729.884345:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985243520:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319729.884378:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319729.884517:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985243392:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319729.885321:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243520 00000100:00100000:3.0:1767319729.885337:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243520 00000100:00100000:3.0:1767319729.885407:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2319 00000100:00100000:3.0:1767319729.885417:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985243520:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319729.885441:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319729.885507:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985243520:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (1128us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319729.885514:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2319 00000100:00080000:0.0:1767319729.885577:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319729.887483:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985243520:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319730.011411:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319730.395336:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319730.395347:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319730.395351:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319730.395355:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319730.395359:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319730.395370:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4973 00000100:00080000:0.0:1767319730.395376:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319730.395383:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319730.395427:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7100 x1853163985243648/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319730.395454:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319730.395458:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319730.395459:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319730.395463:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4973 00000100:00080000:0.0:1767319730.395465:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319730.395469:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319730.395484:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985243776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319730.395530:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319730.395535:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319730.395537:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319730.395540:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4973 00000100:00080000:0.0:1767319730.395543:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319730.395547:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319730.395557:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5180 x1853163985244032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319730.395577:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319730.395580:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319730.395582:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319730.395585:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4973 00000100:00080000:0.0:1767319730.395587:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319730.395590:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319730.395598:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4380 x1853163985244288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319730.395733:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319730.395752:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985243904:0@lo:400:kworker.0 00000100:00100000:1.0:1767319730.395817:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319730.395894:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319730.395902:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985244160:0@lo:400:kworker.0 00000100:00100000:1.0:1767319730.395924:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319730.395955:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243904 00000100:00100000:1.0:1767319730.395962:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243904 00000100:00100000:1.0:1767319730.396002:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244160 00000100:00100000:1.0:1767319730.396005:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244160 00000100:00100000:1.0:1767319730.396022:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4793 00000100:00100000:1.0:1767319730.396030:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985243904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319730.396075:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985243904:12345-0@lo:400:kworker.0 Request processed in 46us (257us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319730.396082:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4793 00000100:00100000:1.0:1767319730.396096:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4794 00000100:00100000:1.0:1767319730.396101:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985244160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319730.396124:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985244160:12345-0@lo:400:kworker.0 Request processed in 22us (199us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319730.396129:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4794 00000100:00100000:1.0:1767319730.396159:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985244160:0@lo:400:kworker.0 00000100:00100000:1.0:1767319730.396210:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985243904:0@lo:400:kworker.0 00000100:00080000:0.0:1767319730.396270:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319730.396723:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985243648:0@lo:38:kworker.0 00000100:00100000:2.0:1767319730.396758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319730.396794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985243776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319730.396821:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319730.396930:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243648 00000100:00100000:0.0:1767319730.396936:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243648 00000100:00100000:0.0:1767319730.396965:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5327 00000100:00100000:0.0:1767319730.396970:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985243648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319730.396988:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319730.397021:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985243648:12345-0@lo:38:kworker.0 Request processed in 49us (261us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319730.397027:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5327 00000100:00100000:3.0:1767319730.397179:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985243776 00000100:00100000:3.0:1767319730.397184:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985243776 00000100:00100000:3.0:1767319730.397217:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1822 00000100:00100000:3.0:1767319730.397223:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985243776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319730.397234:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319730.397277:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985243776:12345-0@lo:38:kworker.0 Request processed in 53us (455us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319730.397284:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1822 00000100:00080000:2.0:1767319730.397760:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319730.397766:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319730.397770:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985243776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319730.397778:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985244032:0@lo:38:kworker.0 00000100:00100000:2.0:1767319730.397809:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319730.398396:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985244288:0@lo:38:kworker.0 00000100:00100000:2.0:1767319730.398423:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319730.403445:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319730.403460:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319730.403465:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985243648:0@lo:38:kworker.0 00000100:00100000:1.0:1767319730.405093:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244032 00000100:00100000:1.0:1767319730.405110:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244032 00000100:00100000:0.0:1767319730.413133:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244288 00000100:00100000:0.0:1767319730.413151:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244288 00000100:00100000:0.0:1767319730.413209:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5328 00000100:00100000:0.0:1767319730.413218:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985244032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319730.413245:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319730.413670:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5329 00000100:00100000:3.0:1767319730.413678:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985244288:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319730.413691:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319730.413775:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985244288:12345-0@lo:38:kworker.0 Request processed in 95us (15349us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319730.413782:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5329 00000100:00080000:2.0:1767319730.415092:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319730.415104:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319730.415109:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985244288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319730.416060:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985244032:12345-0@lo:38:kworker.0 Request processed in 2840us (18248us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319730.416073:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5328 00000100:00080000:3.0:1767319730.417548:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319730.417561:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319730.417566:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985244032:0@lo:38:kworker.0 00010000:00010000:1.0:1767319731.035683:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319731.375383:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319731.375473:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319731.375757:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920577152 00000100:00100000:0.0:1767319731.375768:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920577152 00000100:00080000:2.0:1767319731.375885:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319731.375927:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920577024 00000100:00100000:0.0:1767319731.375931:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920577024 00000100:00100000:3.0:1767319731.382766:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4795 00000100:00100000:3.0:1767319731.382797:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920577152:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319731.382899:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920577152:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 118us (7528us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319731.382908:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4795 00000100:00100000:3.0:1767319731.382950:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5330 00000100:00100000:3.0:1767319731.382955:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920577024:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319731.382977:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319731.383002:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920577024:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (7528us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319731.383009:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5330 00010000:00010000:2.0:1767319732.060488:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319733.083597:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319734.107623:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319735.003787:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1767319735.003804:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985244544:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319735.003881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319735.003938:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244544 00000100:00100000:1.0:1767319735.003946:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244544 00000100:00100000:0.0:1767319735.003960:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985244416:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319735.003983:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319735.004024:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319735.006900:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244416 00000100:00100000:1.0:1767319735.006926:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244416 00000100:00100000:1.0:1767319735.007006:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2320 00000100:00100000:1.0:1767319735.007017:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985244544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319735.007051:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319735.007133:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985244544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 119us (3251us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319735.007139:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2321 00000100:00100000:0.0:1767319735.007148:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985244416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319735.007189:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767319735.007194:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319735.007267:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985244544:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319735.007944:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985244416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 794us (3956us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319735.007996:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2320 00000100:00100000:0.0:1767319735.008000:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2321 00000100:00100000:2.0:1767319735.011285:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985244416:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319735.131980:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319735.515529:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319735.515542:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319735.515545:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319735.515549:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319735.515554:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319735.515563:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4978 00000100:00080000:0.0:1767319735.515568:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319735.515575:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319735.515616:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6d80 x1853163985244672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319735.515642:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319735.515645:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319735.515647:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319735.515649:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4978 00000100:00080000:0.0:1767319735.515651:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319735.515653:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319735.515663:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4000 x1853163985244800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319735.515747:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985244928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319735.515815:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319735.515879:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244928 00000100:00100000:3.0:1767319735.515888:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244928 00000100:00100000:2.0:1767319735.515943:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985244672:0@lo:38:kworker.0 00000100:00100000:3.0:1767319735.515947:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4796 00000100:00100000:3.0:1767319735.515969:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985244928:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319735.515972:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319735.515992:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985244800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319735.516014:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319735.516044:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319735.516048:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319735.516050:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319735.516053:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4978 00000100:00080000:0.0:1767319735.516056:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319735.516059:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319735.516075:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4e00 x1853163985245056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319735.516319:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985245056:0@lo:38:kworker.0 00000100:00100000:2.0:1767319735.516348:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319735.516383:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244800 00000100:00100000:2.0:1767319735.516386:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244800 00000100:00100000:2.0:1767319735.516411:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1823 00000100:00100000:2.0:1767319735.516415:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985244800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319735.516434:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319735.516468:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985244800:12345-0@lo:38:kworker.0 Request processed in 51us (453us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319735.516474:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1823 00000100:00080000:2.0:1767319735.516512:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319735.516517:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319735.516520:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985244800:0@lo:38:kworker.0 00000100:00080000:2.0:1767319735.516558:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319735.516570:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985244672 00000100:00100000:2.0:1767319735.516573:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985244672 00000100:00100000:2.0:1767319735.516592:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245056 00000100:00100000:2.0:1767319735.516594:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245056 00000100:00100000:2.0:1767319735.516608:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5331 00000100:00100000:2.0:1767319735.516613:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985244672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319735.516623:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319735.516651:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985244672:12345-0@lo:38:kworker.0 Request processed in 37us (678us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319735.516657:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5331 00000100:00100000:2.0:1767319735.516668:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5332 00000100:00100000:2.0:1767319735.516672:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985245056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319735.516681:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319735.516703:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985245056:12345-0@lo:38:kworker.0 Request processed in 29us (358us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319735.516708:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5332 00000100:00080000:2.0:1767319735.516728:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319735.516732:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319735.516734:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985245056:0@lo:38:kworker.0 00000100:00080000:2.0:1767319735.516742:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319735.516745:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319735.516747:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985244672:0@lo:38:kworker.0 00000100:00100000:2.0:1767319735.516776:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985245184:0@lo:400:kworker.0 00000100:00100000:2.0:1767319735.516799:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319735.516822:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245184 00000100:00100000:2.0:1767319735.516825:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245184 00000100:00100000:2.0:1767319735.516849:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4797 00000100:00100000:2.0:1767319735.516855:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985245184:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319735.516885:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985245184:12345-0@lo:400:kworker.0 Request processed in 30us (85us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319735.516891:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4797 00000100:00080000:2.0:1767319735.516904:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319735.516922:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985245184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319735.521347:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985244928:12345-0@lo:400:kworker.0 Request processed in 5380us (5519us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319735.521361:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4796 00000100:00100000:1.0:1767319735.521845:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985244928:0@lo:400:kworker.0 00000100:00080000:0.0:1767319735.523660:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319735.523669:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319735.523673:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319735.523680:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4978 00000100:00080000:0.0:1767319735.523684:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319735.523691:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319735.523735:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5180 x1853163985245312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319735.523815:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985245312:0@lo:38:kworker.0 00000100:00100000:2.0:1767319735.523880:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319735.525917:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245312 00000100:00100000:3.0:1767319735.525929:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245312 00000100:00100000:3.0:1767319735.525982:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5333 00000100:00100000:3.0:1767319735.525990:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985245312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319735.526012:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319735.533575:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985245312:12345-0@lo:38:kworker.0 Request processed in 7574us (9684us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319735.533590:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5333 00000100:00080000:2.0:1767319735.537820:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319735.537834:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319735.537839:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985245312:0@lo:38:kworker.0 00010000:00010000:1.0:1767319736.155400:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319736.494974:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319736.495027:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319736.495109:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920577280 00000100:00100000:3.0:1767319736.495123:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920577280 00000100:00100000:3.0:1767319736.495231:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920577408 00000100:00100000:3.0:1767319736.495235:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920577408 00000100:00100000:3.0:1767319736.495337:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4798 00000100:00100000:0.0:1767319736.495546:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5334 00000100:00100000:0.0:1767319736.495564:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920577280:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319736.495592:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319736.495665:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920577280:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 101us (707us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319736.495674:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5334 00000100:00100000:3.0:1767319736.495882:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920577408:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319736.495931:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920577408:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 584us (904us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319736.495939:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4798 00000100:00080000:1.0:1767319736.498435:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319737.181676:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319738.209722:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319739.228845:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319740.123861:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985245440:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319740.123963:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319740.124110:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245440 00000100:00100000:1.0:1767319740.124122:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245440 00000100:00100000:1.0:1767319740.124256:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2322 00000100:00100000:1.0:1767319740.124268:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985245440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319740.124310:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319740.124373:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985245440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319740.124381:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2322 00000100:00100000:1.0:1767319740.124695:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985245440:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319740.124988:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319740.125429:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985245568:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319740.125503:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319740.126054:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245568 00000100:00100000:1.0:1767319740.126063:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245568 00000100:00100000:1.0:1767319740.126156:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2323 00000100:00100000:1.0:1767319740.126202:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985245568:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319740.126238:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767319740.126394:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319740.138707:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985245568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 12527us (13188us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319740.138729:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2323 00000100:00100000:2.0:1767319740.152942:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985245568:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319740.251677:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319740.586406:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319740.586460:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920577536 offset 0 length 520: 4 00000100:00100000:3.0:1767319740.586484:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319740.593858:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920577792 00000100:00100000:0.0:1767319740.593877:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920577792 00000100:00100000:0.0:1767319740.593960:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920577664 00000100:00100000:0.0:1767319740.593964:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920577664 00000100:00100000:0.0:1767319740.593993:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4799 00000100:00100000:0.0:1767319740.594004:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920577792:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319740.594536:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319740.594558:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5335 00000100:00100000:3.0:1767319740.594565:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920577664:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319740.594586:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319740.594635:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920577664:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 69us (8149us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319740.594643:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5335 00000100:00100000:0.0:1767319740.598302:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920577792:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 4287us (11893us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319740.598318:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4799 00000100:00100000:2.0:1767319740.635781:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319715/real 1767319715] req@ffff8ce1b3ff3b80 x1853163985239936/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319740 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319740.635813:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319740.635815:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3ff3b80 x1853163985239936/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319740 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319740.635819:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319740.635822:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319740.635825:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319740.635829:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319740.635834:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4983 00000100:00080000:2.0:1767319740.635834:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319740.635838:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319740.635844:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00080000:2.0:1767319740.635861:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319740.635864:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985239936:0@lo:250:kworker.0 00000100:00100000:0.0:1767319740.635881:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7100 x1853163985245696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319740.635896:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319740.635900:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319740.635901:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319740.635904:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4983 00000100:00100000:2.0:1767319740.635905:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985245696:0@lo:38:kworker.0 00000100:00080000:0.0:1767319740.635907:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319740.635910:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319740.635921:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985245824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319740.635964:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319740.635971:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319740.635974:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319740.635976:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319740.635979:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4983 00000100:00080000:0.0:1767319740.635981:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319740.635984:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319740.635996:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5f80 x1853163985246080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319740.636010:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319740.636013:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319740.636014:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319740.636016:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985245824:0@lo:38:kworker.0 00000100:00080000:0.0:1767319740.636017:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4983 00000100:00080000:0.0:1767319740.636019:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319740.636022:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319740.636032:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6a00 x1853163985246336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319740.636046:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319740.636049:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319740.636051:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319740.636052:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319740.636054:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4963 00000100:00080000:0.0:1767319740.636056:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319740.636059:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319740.636068:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5880 x1853163985246464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319740.636074:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985246080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.636094:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319740.636114:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985246336:0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.636147:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319740.636180:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985246464:0@lo:250:kworker.0 00000400:00000100:2.0:1767319740.636198:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985246464 offset 0 length 520: 4 00000100:00100000:2.0:1767319740.636231:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245824 00000100:00100000:2.0:1767319740.636238:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245824 00000100:00100000:2.0:1767319740.636286:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1824 00000100:00100000:2.0:1767319740.636292:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985245824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319740.636311:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319740.636350:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985245824:12345-0@lo:38:kworker.0 Request processed in 57us (297us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319740.636357:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1824 00000100:00080000:2.0:1767319740.636392:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319740.636396:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985245952:0@lo:400:kworker.0 00000100:00080000:2.0:1767319740.636396:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.636398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985245824:0@lo:38:kworker.0 00000100:00100000:1.0:1767319740.636426:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319740.636457:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245696 00000100:00100000:1.0:1767319740.636461:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245696 00000100:00100000:1.0:1767319740.636484:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985246080 00000100:00100000:1.0:1767319740.636487:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985246080 00000100:00100000:1.0:1767319740.636497:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985246336 00000100:00100000:1.0:1767319740.636499:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985246336 00000100:00100000:1.0:1767319740.636506:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985245952 00000100:00100000:1.0:1767319740.636509:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985245952 00000100:00100000:1.0:1767319740.636534:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4800 00000100:00100000:1.0:1767319740.636539:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985245952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319740.636580:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985245952:12345-0@lo:400:kworker.0 Request processed in 40us (153us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319740.636585:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4800 00000100:00100000:1.0:1767319740.636611:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985245952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319740.636629:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5336 00000100:00100000:1.0:1767319740.636633:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985245696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319740.636643:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319740.636667:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985245696:12345-0@lo:38:kworker.0 Request processed in 33us (703us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319740.636674:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5336 00000100:00100000:1.0:1767319740.636685:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5337 00000100:00100000:1.0:1767319740.636689:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985246080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319740.636697:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319740.636720:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985246080:12345-0@lo:38:kworker.0 Request processed in 30us (625us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319740.636726:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5337 00000100:00100000:1.0:1767319740.636736:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5338 00000100:00100000:1.0:1767319740.636740:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985246336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319740.636748:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319740.636772:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985246336:12345-0@lo:38:kworker.0 Request processed in 30us (624us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319740.636777:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5338 00000100:00080000:3.0:1767319740.636839:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319740.636983:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319740.636988:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.636991:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985246336:0@lo:38:kworker.0 00000100:00080000:2.0:1767319740.636999:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319740.637002:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.637004:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985246080:0@lo:38:kworker.0 00000100:00080000:2.0:1767319740.637011:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319740.637013:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.637016:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985245696:0@lo:38:kworker.0 00000100:00100000:0.0:1767319740.637221:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985246208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319740.637252:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319740.637762:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985246208 00000100:00100000:2.0:1767319740.637768:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985246208 00000100:00100000:2.0:1767319740.637870:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4801 00000100:00100000:2.0:1767319740.637879:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985246208:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319740.637977:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985246208:12345-0@lo:400:kworker.0 Request processed in 100us (724us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319740.637984:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4801 00000100:00100000:1.0:1767319740.638012:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985246208:0@lo:400:kworker.0 00000100:00080000:3.0:1767319740.640800:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319741.276856:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319742.301021:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319743.327547:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319744.347902:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319745.243768:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319745.243789:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985246592:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319745.243866:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319745.243930:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319745.243941:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985246720:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319745.243981:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319745.248639:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985246592 00000100:00100000:2.0:1767319745.248655:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985246592 00000100:00080000:3.0:1767319745.248743:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319745.252757:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985246720 00000100:00100000:0.0:1767319745.252771:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985246720 00000100:00100000:0.0:1767319745.252946:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2324 00000100:00100000:0.0:1767319745.252956:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985246592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319745.252987:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319745.253076:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985246592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 121us (9210us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319745.253084:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2324 00000100:00100000:0.0:1767319745.253104:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2325 00000100:00080000:3.0:1767319745.253108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319745.253114:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985246720:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319745.253131:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319745.253195:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985246720:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (9214us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319745.253202:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2325 00000100:00100000:0.0:1767319745.253250:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985246592:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319745.253252:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985246720:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319745.371388:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319745.706599:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319745.706867:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578048 00000100:00100000:0.0:1767319745.706878:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578048 00000100:00080000:3.0:1767319745.707007:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319745.707121:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4802 00000100:00100000:0.0:1767319745.707133:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920578048:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319745.707258:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920578048:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 127us (668us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319745.707266:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4802 00000100:00100000:3.0:1767319745.707561:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319745.715346:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920577920 00000100:00100000:2.0:1767319745.715360:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920577920 00000100:00100000:2.0:1767319745.715411:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5339 00000100:00100000:2.0:1767319745.715420:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920577920:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319745.715445:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319745.715515:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920577920:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (7953us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319745.715523:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5339 00000100:00080000:2.0:1767319745.755820:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319745.755832:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319745.755835:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319745.755840:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319745.755844:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319745.755853:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4988 00000100:00080000:2.0:1767319745.755858:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319745.755866:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319745.755923:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524dc00 x1853163985246848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319745.755953:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319745.755958:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319745.755959:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319745.755963:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4988 00000100:00080000:2.0:1767319745.755966:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319745.755969:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319745.755983:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4000 x1853163985246976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319745.756070:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985247104:0@lo:400:kworker.0 00000100:00100000:1.0:1767319745.756137:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319745.756243:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319745.756248:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319745.756250:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319745.756254:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4988 00000100:00080000:2.0:1767319745.756257:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319745.756261:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319745.756286:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7100 x1853163985247232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319745.756361:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985246848:0@lo:38:kworker.0 00000100:00100000:3.0:1767319745.756393:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319745.756418:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985246976:0@lo:38:kworker.0 00000100:00100000:3.0:1767319745.756445:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319745.756465:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985247232:0@lo:38:kworker.0 00000100:00100000:3.0:1767319745.756484:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319745.756519:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985247104 00000100:00100000:3.0:1767319745.756527:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985247104 00000100:00100000:3.0:1767319745.756577:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985246848 00000100:00100000:3.0:1767319745.756581:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985246848 00000100:00100000:3.0:1767319745.756596:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985247232 00000100:00100000:3.0:1767319745.756598:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985247232 00000100:00100000:3.0:1767319745.756615:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4803 00000100:00100000:3.0:1767319745.756624:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985247104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319745.757253:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985247360:0@lo:400:kworker.0 00000100:00100000:0.0:1767319745.757292:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319745.760292:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985247360 00000100:00100000:0.0:1767319745.760312:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985247360 00000100:00100000:0.0:1767319745.760385:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4804 00000100:00100000:0.0:1767319745.760396:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985247360:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319745.760501:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985247360:12345-0@lo:400:kworker.0 Request processed in 107us (3208us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319745.760509:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4804 00000100:00100000:0.0:1767319745.760531:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5340 00000100:00100000:0.0:1767319745.760536:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985246848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319745.760559:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319745.760598:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985246848:12345-0@lo:38:kworker.0 Request processed in 60us (4204us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319745.760604:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5340 00000100:00100000:0.0:1767319745.760619:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5341 00000100:00100000:0.0:1767319745.760624:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985247232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319745.760633:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319745.760657:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985247232:12345-0@lo:38:kworker.0 Request processed in 32us (4172us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319745.760664:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5341 00000100:00100000:0.0:1767319745.760703:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985247360:0@lo:400:kworker.0 00000100:00080000:0.0:1767319745.760744:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319745.760749:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319745.760753:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985246848:0@lo:38:kworker.0 00000100:00080000:0.0:1767319745.760763:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319745.760766:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319745.760769:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985247232:0@lo:38:kworker.0 00000100:00080000:0.0:1767319745.760836:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319745.760877:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985246976 00000100:00100000:0.0:1767319745.760881:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985246976 00000100:00100000:0.0:1767319745.760904:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1825 00000100:00100000:0.0:1767319745.760908:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985246976:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319745.760918:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319745.760950:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985246976:12345-0@lo:38:kworker.0 Request processed in 40us (4505us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319745.760956:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1825 00000100:00080000:0.0:1767319745.760980:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319745.760984:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319745.760987:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985246976:0@lo:38:kworker.0 00000100:00080000:2.0:1767319745.763193:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319745.763203:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319745.763207:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319745.763215:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4988 00000100:00080000:2.0:1767319745.763220:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319745.763226:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319745.763266:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73dc00 x1853163985247488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319745.763380:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985247744:0@lo:400:kworker.0 00000100:00100000:2.0:1767319745.763443:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319745.763485:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319745.763490:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985247616:0@lo:400:kworker.0 00000100:00100000:2.0:1767319745.763509:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319745.763536:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985247744 00000100:00100000:2.0:1767319745.763541:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985247744 00000100:00100000:2.0:1767319745.763578:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985247616 00000100:00100000:2.0:1767319745.763580:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985247616 00000100:00100000:2.0:1767319745.763595:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3141 00000100:00100000:2.0:1767319745.763602:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985247744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319745.763641:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985247744:12345-0@lo:400:kworker.0 Request processed in 39us (198us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319745.763647:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3141 00000100:00100000:2.0:1767319745.763660:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3142 00000100:00100000:2.0:1767319745.763665:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985247616:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319745.763683:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985247616:12345-0@lo:400:kworker.0 Request processed in 19us (173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319745.763687:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3142 00000100:00100000:2.0:1767319745.763722:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985247616:0@lo:400:kworker.0 00000100:00100000:2.0:1767319745.763729:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985247744:0@lo:400:kworker.0 00000100:00100000:0.0:1767319745.764041:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985247488:0@lo:38:kworker.0 00000100:00100000:0.0:1767319745.764075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319745.764240:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985247488 00000100:00100000:2.0:1767319745.764244:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985247488 00000100:00100000:2.0:1767319745.764272:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5342 00000100:00100000:2.0:1767319745.764278:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985247488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319745.764297:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319745.764352:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985247488:12345-0@lo:38:kworker.0 Request processed in 73us (276us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319745.764359:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5342 00000100:00080000:2.0:1767319745.764386:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319745.764391:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319745.764394:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985247488:0@lo:38:kworker.0 00000100:00100000:3.0:1767319745.764849:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985247104:12345-0@lo:400:kworker.0 Request processed in 8224us (8709us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319745.764858:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4803 00000100:00100000:1.0:1767319745.768674:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985247104:0@lo:400:kworker.0 00010000:00010000:1.0:1767319746.396077:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319747.419511:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319748.443626:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319749.469629:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319750.363680:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985247872:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319750.363751:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319750.364748:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985247872 00000100:00100000:3.0:1767319750.364759:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985247872 00000100:00100000:3.0:1767319750.364832:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2326 00000100:00100000:3.0:1767319750.364843:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985247872:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319750.364876:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319750.364939:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985247872:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (1187us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319750.364957:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2326 00000100:00080000:0.0:1767319750.365156:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319750.365370:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985247872:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319750.365601:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319750.365613:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985248000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319750.365667:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319750.365721:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248000 00000100:00100000:3.0:1767319750.365727:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248000 00000100:00100000:3.0:1767319750.365779:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2327 00000100:00100000:3.0:1767319750.365787:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985248000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319750.365805:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319750.365838:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985248000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (186us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319750.365845:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2327 00000100:00080000:1.0:1767319750.365904:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319750.365927:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985248000:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319750.491411:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319750.826674:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319750.826768:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578304 00000100:00100000:3.0:1767319750.826779:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578304 00000100:00100000:3.0:1767319750.826856:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4805 00000100:00100000:3.0:1767319750.826867:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920578304:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319750.826980:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920578304:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 115us (316us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319750.826989:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4805 00000100:00080000:1.0:1767319750.827118:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319750.827877:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319750.830335:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578176 00000100:00100000:0.0:1767319750.830349:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578176 00000100:00100000:0.0:1767319750.830414:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5343 00000100:00100000:0.0:1767319750.830422:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920578176:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319750.830446:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319750.830523:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920578176:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 99us (2655us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319750.830530:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5343 00000100:00080000:1.0:1767319750.875418:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.875429:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319750.875432:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.875436:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.875440:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.875448:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.875454:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319750.875460:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319750.875502:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407b80 x1853163985248128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319750.875529:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.875533:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.875535:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.875537:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.875540:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319750.875543:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319750.875556:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985248256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319750.875602:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.875606:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.875608:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.875611:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.875613:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319750.875616:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319750.875627:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405f80 x1853163985248512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319750.875643:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.875646:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.875647:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.875651:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.875653:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319750.875656:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319750.875668:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404380 x1853163985248768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319750.875718:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985249024:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.875783:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319750.875845:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319750.875852:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985248896:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.875874:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319750.875905:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249024 00000100:00100000:1.0:1767319750.875911:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249024 00000100:00100000:1.0:1767319750.875959:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248896 00000100:00100000:1.0:1767319750.875963:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248896 00000100:00100000:1.0:1767319750.875988:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3143 00000100:00100000:1.0:1767319750.875995:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985249024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319750.876032:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985248384:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.876048:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985249024:12345-0@lo:400:kworker.0 Request processed in 52us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319750.876056:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3143 00000100:00100000:3.0:1767319750.876072:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319750.876076:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3144 00000100:00100000:1.0:1767319750.876085:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985248896:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.876118:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985248896:12345-0@lo:400:kworker.0 Request processed in 34us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319750.876123:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3144 00000100:00100000:1.0:1767319750.876191:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985248896:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.876202:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985249024:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.876245:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248384 00000100:00100000:1.0:1767319750.876248:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248384 00000100:00100000:1.0:1767319750.876284:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4806 00000100:00100000:1.0:1767319750.876291:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985248384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319750.876298:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319750.876307:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985248640:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.876331:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985248384:12345-0@lo:400:kworker.0 Request processed in 40us (260us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319750.876343:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319750.876347:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4806 00000100:00100000:1.0:1767319750.876361:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248640 00000100:00100000:1.0:1767319750.876364:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248640 00000100:00100000:3.0:1767319750.876385:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985248384:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.876391:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4807 00000100:00100000:1.0:1767319750.876401:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985248640:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319750.876420:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319750.876441:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985248640:12345-0@lo:400:kworker.0 Request processed in 42us (101us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319750.876449:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985248640:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.876450:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4807 00000100:00100000:2.0:1767319750.876715:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985248128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.876747:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319750.876784:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985248256:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.876812:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319750.876844:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985248512:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.876867:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319750.876888:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985248768:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.876911:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319750.876911:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248128 00000100:00100000:3.0:1767319750.876917:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248128 00000100:00100000:3.0:1767319750.876950:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248512 00000100:00100000:3.0:1767319750.876953:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248512 00000100:00100000:2.0:1767319750.876962:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248768 00000100:00100000:2.0:1767319750.876968:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248768 00000100:00100000:3.0:1767319750.876979:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5344 00000100:00100000:3.0:1767319750.876986:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985248128:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.876996:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5345 00000100:00100000:2.0:1767319750.877001:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985248512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319750.877018:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319750.877053:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985248512:12345-0@lo:38:kworker.0 Request processed in 50us (185us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319750.877059:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5345 00000100:00100000:2.0:1767319750.877073:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5346 00000100:00100000:2.0:1767319750.877077:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985248768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319750.877080:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767319750.877092:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319750.877122:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985248768:12345-0@lo:38:kworker.0 Request processed in 44us (211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319750.877129:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5346 00000100:00080000:2.0:1767319750.877156:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.877161:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.877187:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985248512:0@lo:38:kworker.0 00000100:00080000:2.0:1767319750.877196:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.877199:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.877202:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985248768:0@lo:38:kworker.0 00000100:00100000:3.0:1767319750.877276:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985248128:12345-0@lo:38:kworker.0 Request processed in 288us (526us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319750.877284:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5344 00000100:00080000:2.0:1767319750.877549:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.877555:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.877558:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985248128:0@lo:38:kworker.0 00000100:00100000:1.0:1767319750.877606:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985248256 00000100:00100000:1.0:1767319750.877611:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985248256 00000100:00100000:1.0:1767319750.877642:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1826 00000100:00100000:1.0:1767319750.877647:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985248256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319750.877659:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319750.877701:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985248256:12345-0@lo:38:kworker.0 Request processed in 53us (889us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319750.877707:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1826 00000100:00080000:2.0:1767319750.877864:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.877869:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.877872:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985248256:0@lo:38:kworker.0 00010000:00010000:1.0:1767319751.515980:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319752.539411:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319753.564469:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319754.587424:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319755.483545:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985249280:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319755.483545:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985249152:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319755.483613:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.483718:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319755.483809:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249152 00000100:00100000:2.0:1767319755.483817:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249152 00000100:00100000:2.0:1767319755.483894:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249280 00000100:00100000:0.0:1767319755.483926:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2328 00000100:00100000:0.0:1767319755.483938:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985249152:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319755.483979:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319755.484015:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249280 00000100:00100000:0.0:1767319755.484026:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985249152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 91us (414us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319755.484034:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2328 00000100:00100000:0.0:1767319755.484053:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2329 00000100:00100000:0.0:1767319755.484059:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985249280:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319755.484080:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767319755.484089:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319755.484094:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319755.484116:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985249280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 57us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319755.484370:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985249152:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319755.484405:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985249280:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319755.489509:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2329 00010000:00010000:1.0:1767319755.611687:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319755.963723:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319755.963776:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319755.964199:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578560 00000100:00100000:3.0:1767319755.964211:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578560 00000100:00100000:3.0:1767319755.964289:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578432 00000100:00100000:3.0:1767319755.964293:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578432 00000100:00100000:3.0:1767319755.964313:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4808 00000100:00100000:3.0:1767319755.964322:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920578560:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319755.964402:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920578560:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (689us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319755.964411:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4808 00000100:00100000:3.0:1767319755.964432:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5347 00000100:00100000:3.0:1767319755.964437:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920578432:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319755.964458:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319755.964481:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920578432:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (705us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319755.964487:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5347 00000100:00080000:1.0:1767319755.964804:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319755.995635:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.995646:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319755.995649:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.995662:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.995667:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.995676:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.995681:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.995688:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319755.995732:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98a80 x1853163985249408/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319755.995757:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.995760:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.995762:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.995765:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.995768:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.995772:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319755.995784:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9b100 x1853163985249536/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319755.995828:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.995832:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.995834:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.995838:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.995848:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.995852:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319755.995863:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631c00 x1853163985249792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319755.995877:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.995880:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.995882:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.995885:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.995888:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.995891:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319755.995901:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633b80 x1853163985250048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319755.996246:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985249664:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.996307:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.996373:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319755.996379:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985249920:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.996400:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.996424:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985250304:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.996448:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.996467:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319755.996472:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985250176:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.996491:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.996518:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250304 00000100:00100000:1.0:1767319755.996525:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250304 00000100:00100000:1.0:1767319755.996564:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250176 00000100:00100000:1.0:1767319755.996569:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250176 00000100:00100000:1.0:1767319755.996587:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3145 00000100:00100000:1.0:1767319755.996595:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985250304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.996655:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985250304:12345-0@lo:400:kworker.0 Request processed in 60us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319755.996663:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3145 00000100:00100000:1.0:1767319755.996679:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3146 00000100:00100000:1.0:1767319755.996685:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985250176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.996712:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985250176:12345-0@lo:400:kworker.0 Request processed in 27us (220us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319755.996717:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3146 00000100:00100000:1.0:1767319755.996773:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985250176:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.996782:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985250304:0@lo:400:kworker.0 00000100:00100000:2.0:1767319755.996937:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985249408:0@lo:38:kworker.0 00000100:00100000:2.0:1767319755.996968:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319755.996996:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985249536:0@lo:38:kworker.0 00000100:00100000:2.0:1767319755.997029:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319755.997055:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985249792:0@lo:38:kworker.0 00000100:00100000:2.0:1767319755.997075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319755.997094:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985250048:0@lo:38:kworker.0 00000100:00100000:2.0:1767319755.997113:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319755.997144:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249536 00000100:00100000:2.0:1767319755.997148:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249536 00000100:00100000:2.0:1767319755.997192:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1827 00000100:00100000:2.0:1767319755.997197:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985249536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319755.997217:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319755.997251:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985249536:12345-0@lo:38:kworker.0 Request processed in 53us (222us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319755.997257:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1827 00000100:00100000:2.0:1767319755.997277:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249664 00000100:00100000:2.0:1767319755.997280:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249664 00000100:00100000:2.0:1767319755.997312:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249920 00000100:00100000:2.0:1767319755.997315:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249920 00000100:00100000:2.0:1767319755.997327:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249408 00000100:00100000:2.0:1767319755.997330:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249408 00000100:00100000:2.0:1767319755.997338:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985249792 00000100:00100000:2.0:1767319755.997340:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985249792 00000100:00100000:2.0:1767319755.997346:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250048 00000100:00100000:2.0:1767319755.997348:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250048 00000100:00100000:2.0:1767319755.997358:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4809 00000100:00100000:2.0:1767319755.997364:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985249664:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319755.997400:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985249664:12345-0@lo:400:kworker.0 Request processed in 36us (1093us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319755.997407:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4809 00000100:00100000:2.0:1767319755.997418:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4810 00000100:00100000:2.0:1767319755.997424:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985249920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319755.997449:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985249920:12345-0@lo:400:kworker.0 Request processed in 25us (1048us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319755.997454:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4810 00000100:00100000:2.0:1767319755.997465:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5348 00000100:00100000:2.0:1767319755.997469:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985249408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319755.997478:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319755.997498:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985249408:12345-0@lo:38:kworker.0 Request processed in 28us (530us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319755.997505:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5348 00000100:00100000:2.0:1767319755.997515:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5349 00000100:00100000:2.0:1767319755.997519:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985249792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319755.997527:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319755.997547:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985249792:12345-0@lo:38:kworker.0 Request processed in 27us (471us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319755.997552:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5349 00000100:00100000:2.0:1767319755.997561:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5350 00000100:00100000:2.0:1767319755.997565:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985250048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319755.997574:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319755.997603:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985250048:12345-0@lo:38:kworker.0 Request processed in 26us (478us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319755.997610:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5350 00000100:00080000:2.0:1767319755.997630:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319755.997654:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985249920:0@lo:400:kworker.0 00000100:00100000:2.0:1767319755.997664:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985249664:0@lo:400:kworker.0 00000100:00080000:2.0:1767319755.997692:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319755.997697:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319755.997700:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985249408:0@lo:38:kworker.0 00000100:00080000:2.0:1767319755.997707:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319755.997710:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319755.997713:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985249536:0@lo:38:kworker.0 00000100:00080000:2.0:1767319755.997720:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319755.997723:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319755.997727:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985249792:0@lo:38:kworker.0 00000100:00080000:2.0:1767319755.997734:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319755.997736:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319755.997739:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985250048:0@lo:38:kworker.0 00010000:00010000:1.0:1767319756.635394:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319757.667785:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319758.683555:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319759.708489:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319760.603741:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985250432:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319760.603846:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319760.603979:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250432 00000100:00100000:0.0:1767319760.603990:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250432 00000100:00100000:0.0:1767319760.604059:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2330 00000100:00100000:0.0:1767319760.604069:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985250432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319760.604099:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319760.604151:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985250432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (304us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319760.604159:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2330 00000100:00100000:0.0:1767319760.604233:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985250432:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319760.604440:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319760.609371:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319760.609398:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985250560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319760.609473:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319760.609575:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250560 00000100:00100000:2.0:1767319760.609584:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250560 00000100:00100000:2.0:1767319760.609650:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2331 00000100:00100000:2.0:1767319760.609658:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985250560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319760.609687:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319760.609733:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985250560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (259us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319760.609741:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2331 00000100:00100000:0.0:1767319760.609848:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985250560:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319760.610904:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319760.731372:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319761.066041:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319761.066157:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578816 00000100:00100000:1.0:1767319761.066199:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578816 00000100:00100000:1.0:1767319761.066281:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4811 00000100:00100000:1.0:1767319761.066292:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920578816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319761.066379:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920578816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319761.066387:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4811 00000100:00080000:2.0:1767319761.066785:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319761.073737:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319761.073859:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578688 00000100:00100000:3.0:1767319761.073870:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578688 00000100:00100000:3.0:1767319761.073921:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5351 00000100:00100000:3.0:1767319761.073930:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920578688:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319761.073953:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319761.074030:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920578688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 99us (302us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319761.074037:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5351 00000100:00080000:3.0:1767319761.115503:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319761.115514:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319761.115517:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319761.115521:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319761.115526:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319761.115534:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5004 00000100:00080000:3.0:1767319761.115538:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319761.115544:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319761.115588:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631880 x1853163985250688/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319761.115612:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319761.115616:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319761.115618:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319761.115622:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5004 00000100:00080000:3.0:1767319761.115624:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319761.115627:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319761.115640:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632300 x1853163985250816/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319761.115729:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767319761.115756:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319761.115758:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985250688:0@lo:38:kworker.0 00000100:00080000:3.0:1767319761.115762:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319761.115766:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319761.115771:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5004 00000100:00080000:3.0:1767319761.115774:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319761.115777:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319761.115792:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985251072/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319761.115838:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319761.115870:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319761.115883:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985251200:0@lo:400:kworker.0 00000100:00100000:2.0:1767319761.115896:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985250816:0@lo:38:kworker.0 00000100:00100000:0.0:1767319761.115921:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319761.115929:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319761.115970:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250688 00000100:00100000:2.0:1767319761.115970:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985251072:0@lo:38:kworker.0 00000100:00100000:0.0:1767319761.115979:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250688 00000100:00100000:2.0:1767319761.115995:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319761.116026:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250816 00000100:00100000:2.0:1767319761.116029:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250816 00000100:00100000:1.0:1767319761.116043:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985250944:0@lo:400:kworker.0 00000100:00100000:2.0:1767319761.116056:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1828 00000100:00100000:2.0:1767319761.116064:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985250816:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319761.116076:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767319761.116085:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319761.116123:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985250816:12345-0@lo:38:kworker.0 Request processed in 58us (193us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319761.116129:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1828 00000100:00080000:3.0:1767319761.120561:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319761.120572:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319761.120576:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319761.120584:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5004 00000100:00080000:3.0:1767319761.120589:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319761.120596:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319761.120632:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632680 x1853163985251328/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319761.120723:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985251584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319761.120798:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319761.120837:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319761.120844:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985251456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319761.120864:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319761.120894:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985251584 00000100:00100000:3.0:1767319761.120901:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985251584 00000100:00100000:3.0:1767319761.120943:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985251456 00000100:00100000:3.0:1767319761.120946:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985251456 00000100:00100000:3.0:1767319761.120968:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3147 00000100:00100000:3.0:1767319761.120975:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985251584:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319761.121041:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985251584:12345-0@lo:400:kworker.0 Request processed in 65us (241us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319761.121048:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3147 00000100:00100000:3.0:1767319761.121066:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3148 00000100:00100000:3.0:1767319761.121072:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985251456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319761.121098:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985251456:12345-0@lo:400:kworker.0 Request processed in 27us (233us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319761.121103:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3148 00000100:00100000:3.0:1767319761.121159:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985251456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319761.121416:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985251584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319761.121546:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985251328:0@lo:38:kworker.0 00000100:00100000:0.0:1767319761.127828:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985251200 00000100:00100000:0.0:1767319761.127845:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985251200 00000100:00100000:0.0:1767319761.127936:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985251072 00000100:00100000:0.0:1767319761.127940:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985251072 00000100:00100000:0.0:1767319761.127951:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985250944 00000100:00100000:0.0:1767319761.127954:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985250944 00000100:00100000:0.0:1767319761.127971:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4812 00000100:00100000:0.0:1767319761.127980:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985251200:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319761.128992:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985251200:12345-0@lo:400:kworker.0 Request processed in 1007us (13065us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319761.129003:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4812 00000100:00100000:0.0:1767319761.129371:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4813 00000100:00100000:0.0:1767319761.129383:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985250944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319761.129436:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985250944:12345-0@lo:400:kworker.0 Request processed in 58us (13360us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319761.129446:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4813 00000100:00100000:0.0:1767319761.129459:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5352 00000100:00100000:0.0:1767319761.129463:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985250688:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319761.129482:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319761.129511:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985250688:12345-0@lo:38:kworker.0 Request processed in 47us (13678us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319761.129517:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5352 00000100:00100000:0.0:1767319761.129527:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5353 00000100:00100000:0.0:1767319761.129531:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985251072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319761.129539:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319761.129560:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985251072:12345-0@lo:38:kworker.0 Request processed in 27us (13565us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319761.129565:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5353 00000100:00100000:0.0:1767319761.129706:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319761.129766:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319761.129771:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319761.129776:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985251072:0@lo:38:kworker.0 00000100:00080000:0.0:1767319761.129788:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319761.129792:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319761.129795:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985250688:0@lo:38:kworker.0 00000100:00080000:0.0:1767319761.129805:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319761.129808:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319761.129811:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985250816:0@lo:38:kworker.0 00000100:00100000:0.0:1767319761.129847:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985251328 00000100:00100000:0.0:1767319761.129852:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985251328 00000100:00100000:0.0:1767319761.129878:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5354 00000100:00100000:0.0:1767319761.129882:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985251328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319761.129891:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319761.129921:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985251328:12345-0@lo:38:kworker.0 Request processed in 38us (215us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319761.129928:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5354 00000100:00080000:0.0:1767319761.129950:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319761.129954:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319761.129956:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985251328:0@lo:38:kworker.0 00000100:00100000:0.0:1767319761.129992:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985250944:0@lo:400:kworker.0 00000100:00080000:0.0:1767319761.130328:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319761.130365:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985251200:0@lo:400:kworker.0 00010000:00010000:1.0:1767319761.755416:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319762.779410:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319763.803794:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319764.827423:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319765.211275:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319740/real 1767319740] req@ffff8ce19e9c5880 x1853163985246464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319765 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319765.211304:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19e9c5880 x1853163985246464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319765 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319765.211319:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319765.211345:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319765.211349:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985246464:0@lo:250:kworker.0 00000100:00080000:3.0:1767319765.211427:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319765.211432:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319765.211435:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319765.211440:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 4988 00000100:00080000:3.0:1767319765.211443:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319765.211450:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319765.211481:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630380 x1853163985251712/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319765.211507:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319765.211518:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319765.211520:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319765.211523:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5009 00000100:00080000:3.0:1767319765.211526:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319765.211529:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319765.211543:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632d80 x1853163985251840/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319765.211551:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319765.211555:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319765.211556:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319765.211559:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5009 00000100:00080000:3.0:1767319765.211562:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319765.211564:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319765.211573:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630e00 x1853163985251968/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319765.211611:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319765.211615:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319765.211616:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319765.211620:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5009 00000100:00080000:3.0:1767319765.211622:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319765.211625:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319765.211632:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985252096:0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.211654:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630700 x1853163985252224/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319765.211701:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319765.211705:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319765.211708:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319765.211710:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319765.211713:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5009 00000100:00080000:3.0:1767319765.211715:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767319765.211719:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319765.211730:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633800 x1853163985252480/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319765.211764:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319765.211770:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985252352:0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.211770:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985252736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319765.211791:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319765.211795:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319765.211810:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319765.211815:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985252608:0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.211833:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319765.211834:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252096 00000100:00100000:0.0:1767319765.211840:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252096 00000100:00100000:3.0:1767319765.211856:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252736 00000100:00100000:3.0:1767319765.211859:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252736 00000100:00100000:3.0:1767319765.211883:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252608 00000100:00100000:0.0:1767319765.211885:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252352 00000100:00100000:3.0:1767319765.211885:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252608 00000100:00100000:0.0:1767319765.211887:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252352 00000100:00100000:0.0:1767319765.211905:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4814 00000100:00100000:3.0:1767319765.211905:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3149 00000100:00100000:0.0:1767319765.211912:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985252096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.211912:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985252736:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319765.211956:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985252096:12345-0@lo:400:kworker.0 Request processed in 44us (254us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319765.211956:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985252736:12345-0@lo:400:kworker.0 Request processed in 44us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319765.211963:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3149 00000100:00100000:0.0:1767319765.211971:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4814 00000100:00100000:3.0:1767319765.211980:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3150 00000100:00100000:0.0:1767319765.211986:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4815 00000100:00100000:3.0:1767319765.211986:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985252608:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319765.211990:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985252352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.212010:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985252608:12345-0@lo:400:kworker.0 Request processed in 25us (175us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319765.212014:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985252352:12345-0@lo:400:kworker.0 Request processed in 24us (222us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319765.212016:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3150 00000100:00100000:0.0:1767319765.212019:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4815 00000100:00080000:0.0:1767319765.212030:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319765.212034:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319765.212036:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319765.212052:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985252352:0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.212052:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985252608:0@lo:400:kworker.0 00000100:00100000:0.0:1767319765.212061:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985252096:0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.212061:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985252736:0@lo:400:kworker.0 00000100:00100000:2.0:1767319765.212408:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985251712:0@lo:250:kworker.0 00000400:00000100:2.0:1767319765.212431:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985251712 offset 0 length 520: 4 00000100:00100000:2.0:1767319765.212449:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985251840:0@lo:38:kworker.0 00000100:00100000:2.0:1767319765.212465:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319765.212488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985251968:0@lo:38:kworker.0 00000100:00100000:2.0:1767319765.212517:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319765.212537:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985252224:0@lo:38:kworker.0 00000100:00100000:2.0:1767319765.212557:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319765.212574:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985252480:0@lo:38:kworker.0 00000100:00100000:0.0:1767319765.212581:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985251840 00000100:00100000:0.0:1767319765.212585:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985251840 00000100:00100000:2.0:1767319765.212591:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319765.212648:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985251968 00000100:00100000:3.0:1767319765.212652:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985251968 00000100:00100000:3.0:1767319765.212689:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1829 00000100:00100000:3.0:1767319765.212694:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985251968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319765.212711:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319765.212746:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985251968:12345-0@lo:38:kworker.0 Request processed in 51us (237us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319765.212752:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1829 00000100:00080000:3.0:1767319765.212773:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319765.212777:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319765.212780:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985251968:0@lo:38:kworker.0 00000100:00100000:0.0:1767319765.217008:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252224 00000100:00100000:0.0:1767319765.217025:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252224 00000100:00100000:0.0:1767319765.217054:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252480 00000100:00100000:0.0:1767319765.217057:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252480 00000100:00100000:0.0:1767319765.217075:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5355 00000100:00100000:0.0:1767319765.217083:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985251840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319765.217109:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319765.217211:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985251840:12345-0@lo:38:kworker.0 Request processed in 127us (4743us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319765.217218:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5355 00000100:00100000:0.0:1767319765.217235:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5356 00000100:00100000:0.0:1767319765.217240:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985252224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319765.217249:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319765.217273:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985252224:12345-0@lo:38:kworker.0 Request processed in 32us (4716us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319765.217279:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5356 00000100:00100000:0.0:1767319765.217291:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5357 00000100:00100000:0.0:1767319765.217295:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985252480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319765.217303:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319765.217325:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985252480:12345-0@lo:38:kworker.0 Request processed in 29us (4733us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319765.217331:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5357 00000100:00080000:0.0:1767319765.217416:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319765.217422:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319765.217426:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985251840:0@lo:38:kworker.0 00000100:00080000:0.0:1767319765.217435:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319765.217439:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319765.217442:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985252224:0@lo:38:kworker.0 00000100:00080000:0.0:1767319765.217450:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319765.217453:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319765.217456:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985252480:0@lo:38:kworker.0 00000100:00100000:1.0:1767319765.723945:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985252864:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319765.724012:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319765.724142:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252864 00000100:00100000:0.0:1767319765.724150:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252864 00000100:00100000:0.0:1767319765.724215:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2332 00000100:00100000:0.0:1767319765.724225:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985252864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319765.724262:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319765.724314:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985252864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (301us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319765.724321:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2332 00000100:00100000:3.0:1767319765.724397:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985252992:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319765.724427:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319765.724619:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985252864:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319765.727351:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985252992 00000100:00100000:0.0:1767319765.727367:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985252992 00000100:00100000:0.0:1767319765.727502:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2333 00000100:00100000:0.0:1767319765.727513:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985252992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319765.727547:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319765.727634:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985252992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 123us (3205us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319765.727641:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2333 00000100:00100000:1.0:1767319765.727739:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985252992:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319765.851393:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319766.189947:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319766.189991:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767319766.190020:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920579200 offset 0 length 520: 4 00000100:00100000:1.0:1767319766.190311:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920579072 00000100:00100000:1.0:1767319766.190322:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920579072 00000100:00100000:3.0:1767319766.190359:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920578944 00000100:00100000:3.0:1767319766.190364:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920578944 00000100:00100000:3.0:1767319766.190405:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4816 00000100:00100000:3.0:1767319766.190416:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920579072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767319766.190420:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319766.190511:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920579072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 97us (572us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319766.190523:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4816 00000100:00100000:3.0:1767319766.190541:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5358 00000100:00100000:3.0:1767319766.190546:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920578944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319766.190567:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319766.190598:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920578944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (599us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319766.190604:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5358 00010000:00010000:1.0:1767319766.875499:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319767.899449:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319768.924409:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319769.947341:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319770.331708:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319770.331717:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319770.331720:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319770.331723:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319770.331726:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319770.331734:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5013 00000100:00080000:0.0:1767319770.331740:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1767319770.331746:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319770.331794:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4380 x1853163985253120/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319770.331822:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319770.331826:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319770.331828:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319770.331831:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5013 00000100:00080000:0.0:1767319770.331833:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1767319770.331837:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319770.331851:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985253248/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319770.331887:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319770.331891:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319770.331892:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319770.331896:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5013 00000100:00080000:0.0:1767319770.331898:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1767319770.331901:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319770.331913:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7800 x1853163985253504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319770.331929:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319770.331932:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319770.331934:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319770.331937:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5013 00000100:00080000:0.0:1767319770.331940:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1767319770.331943:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319770.331953:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4000 x1853163985253760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319770.332002:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985253632:0@lo:400:kworker.0 00000100:00100000:2.0:1767319770.332015:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985253376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319770.332150:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985253888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319770.332234:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319770.332246:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985253120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319770.332279:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319770.332350:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985254016:0@lo:400:kworker.0 00000100:00100000:1.0:1767319770.332382:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319770.332416:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254016 00000100:00100000:1.0:1767319770.332423:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254016 00000100:00100000:1.0:1767319770.332477:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3151 00000100:00100000:1.0:1767319770.332488:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985254016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319770.332500:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985253376 00000100:00100000:2.0:1767319770.332506:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985253376 00000100:00100000:2.0:1767319770.332540:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985253120 00000100:00100000:1.0:1767319770.332541:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985254016:12345-0@lo:400:kworker.0 Request processed in 55us (156us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319770.332545:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985253120 00000100:00100000:1.0:1767319770.332553:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3151 00000100:00100000:2.0:1767319770.332570:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4817 00000100:00080000:1.0:1767319770.332589:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319770.332594:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319770.332624:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985254016:0@lo:400:kworker.0 00000100:00100000:1.0:1767319770.332655:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319770.332679:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985253632 00000100:00100000:1.0:1767319770.332682:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985253632 00000100:00100000:2.0:1767319770.332848:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985253376:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319770.332901:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985253376:12345-0@lo:400:kworker.0 Request processed in 57us (668us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319770.333271:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319770.333305:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4817 00000100:00100000:2.0:1767319770.333325:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4818 00000100:00100000:2.0:1767319770.333333:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985253632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319770.333369:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985253632:12345-0@lo:400:kworker.0 Request processed in 37us (712us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319770.333374:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4818 00000100:00100000:2.0:1767319770.333385:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5359 00000100:00100000:2.0:1767319770.333388:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985253120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319770.333404:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319770.333523:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985253120:12345-0@lo:38:kworker.0 Request processed in 132us (1244us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319770.333530:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5359 00000100:00100000:2.0:1767319770.333566:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985253376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319770.333825:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985253888 00000100:00100000:3.0:1767319770.333830:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985253888 00000100:00100000:3.0:1767319770.333869:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3152 00000100:00100000:3.0:1767319770.333875:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985253888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319770.333912:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985253888:12345-0@lo:400:kworker.0 Request processed in 37us (641us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319770.333917:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3152 00000100:00100000:3.0:1767319770.333945:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985253888:0@lo:400:kworker.0 00000100:00080000:3.0:1767319770.333972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319770.333976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319770.333992:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985253632:0@lo:400:kworker.0 00000100:00080000:0.0:1767319770.335069:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319770.335078:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319770.335082:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985253120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319770.335093:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985253248:0@lo:38:kworker.0 00000100:00100000:0.0:1767319770.335448:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319770.335534:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985253504:0@lo:38:kworker.0 00000100:00100000:0.0:1767319770.335557:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319770.335587:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985253760:0@lo:38:kworker.0 00000100:00100000:0.0:1767319770.335605:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319770.337244:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985253504 00000100:00100000:1.0:1767319770.337256:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985253504 00000100:00100000:1.0:1767319770.337302:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985253760 00000100:00100000:1.0:1767319770.337305:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985253760 00000100:00100000:1.0:1767319770.337334:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5360 00000100:00100000:1.0:1767319770.337341:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985253504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319770.337364:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319770.337428:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985253504:12345-0@lo:38:kworker.0 Request processed in 86us (1869us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319770.337434:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5360 00000100:00100000:1.0:1767319770.337448:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5361 00000100:00100000:1.0:1767319770.337451:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985253760:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319770.337460:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319770.337478:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985253760:12345-0@lo:38:kworker.0 Request processed in 25us (1871us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319770.337483:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5361 00000100:00080000:0.0:1767319770.338091:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319770.338098:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319770.338102:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985253504:0@lo:38:kworker.0 00000100:00080000:0.0:1767319770.338124:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319770.338128:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319770.338131:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985253760:0@lo:38:kworker.0 00000100:00100000:2.0:1767319770.342561:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985253248 00000100:00100000:2.0:1767319770.342583:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985253248 00000100:00100000:2.0:1767319770.342709:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1830 00000100:00100000:2.0:1767319770.342717:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985253248:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319770.342743:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319770.345559:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985253248:12345-0@lo:38:kworker.0 Request processed in 2833us (10102us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319770.345570:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1830 00000100:00080000:0.0:1767319770.346059:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319770.346065:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319770.346069:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985253248:0@lo:38:kworker.0 00000100:00100000:1.0:1767319770.843946:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319770.843969:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985254144:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319770.844048:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319770.844235:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254144 00000100:00100000:0.0:1767319770.844246:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254144 00000100:00100000:0.0:1767319770.844299:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2334 00000100:00100000:0.0:1767319770.844308:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985254144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319770.844336:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319770.844378:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985254144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (330us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319770.844385:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2334 00000100:00100000:0.0:1767319770.844433:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985254144:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319770.844865:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985254272:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319770.844894:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319770.844946:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254272 00000100:00100000:1.0:1767319770.844951:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254272 00000100:00100000:1.0:1767319770.844982:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2335 00000100:00100000:1.0:1767319770.844995:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985254272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319770.845011:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319770.845746:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985254272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 755us (848us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319770.845754:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2335 00000100:00100000:0.0:1767319770.845843:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985254272:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319770.972138:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319771.310902:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319771.310944:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319771.311020:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920579456 00000100:00100000:3.0:1767319771.311030:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920579456 00000100:00100000:3.0:1767319771.311104:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920579328 00000100:00100000:3.0:1767319771.311108:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920579328 00000100:00100000:3.0:1767319771.311128:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4819 00000100:00100000:3.0:1767319771.311139:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920579456:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319771.311276:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5362 00000100:00100000:0.0:1767319771.311282:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920579328:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319771.311303:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319771.311359:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920579328:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 76us (414us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319771.311365:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5362 00000100:00080000:1.0:1767319771.311627:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319771.311727:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920579456:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 589us (834us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319771.311786:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4819 00010000:00010000:2.0:1767319771.997932:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319773.019671:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319774.047748:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319775.067475:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319775.453107:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319775.453117:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319775.453120:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319775.453124:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319775.453127:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319775.453145:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5018 00000100:00080000:2.0:1767319775.453151:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767319775.453157:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319775.453230:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7800 x1853163985254400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319775.453257:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319775.453260:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319775.453262:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319775.453265:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5018 00000100:00080000:2.0:1767319775.453267:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767319775.453271:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319775.453282:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4380 x1853163985254528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319775.453345:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985254656:0@lo:400:kworker.0 00000100:00100000:3.0:1767319775.453422:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319775.453540:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254656 00000100:00100000:1.0:1767319775.453549:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254656 00000100:00080000:2.0:1767319775.453565:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319775.453570:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319775.453572:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319775.453575:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5018 00000100:00080000:2.0:1767319775.453579:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767319775.453583:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319775.453600:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985254784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319775.453607:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4820 00000100:00100000:1.0:1767319775.453621:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985254656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319775.453702:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985254400:0@lo:38:kworker.0 00000100:00100000:0.0:1767319775.453788:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319775.453826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985254528:0@lo:38:kworker.0 00000100:00100000:0.0:1767319775.453857:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319775.453888:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985254784:0@lo:38:kworker.0 00000100:00100000:0.0:1767319775.453912:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319775.455778:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254400 00000100:00100000:0.0:1767319775.455797:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254400 00000100:00100000:0.0:1767319775.455851:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254784 00000100:00100000:0.0:1767319775.455855:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254784 00000100:00100000:0.0:1767319775.455883:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5363 00000100:00100000:0.0:1767319775.455892:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985254400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319775.455921:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319775.456209:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985254400:12345-0@lo:38:kworker.0 Request processed in 315us (2419us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319775.456217:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5363 00000100:00100000:0.0:1767319775.456240:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5364 00000100:00100000:0.0:1767319775.456245:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985254784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319775.456256:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319775.456282:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985254784:12345-0@lo:38:kworker.0 Request processed in 36us (2370us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319775.456287:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5364 00000100:00080000:0.0:1767319775.456340:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319775.456347:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319775.456351:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985254400:0@lo:38:kworker.0 00000100:00080000:0.0:1767319775.456364:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319775.456368:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319775.456371:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985254784:0@lo:38:kworker.0 00000100:00080000:0.0:1767319775.456430:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319775.456462:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254528 00000100:00100000:0.0:1767319775.456468:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254528 00000100:00100000:0.0:1767319775.456513:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1831 00000100:00100000:0.0:1767319775.456518:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985254528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319775.456528:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319775.456562:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985254528:12345-0@lo:38:kworker.0 Request processed in 42us (2704us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319775.456568:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1831 00000100:00100000:0.0:1767319775.456592:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985254912:0@lo:400:kworker.0 00000100:00100000:0.0:1767319775.456632:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319775.456670:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985254912 00000100:00100000:0.0:1767319775.456674:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985254912 00000100:00100000:0.0:1767319775.457285:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4821 00000100:00100000:0.0:1767319775.457297:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985254912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319775.457354:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985254912:12345-0@lo:400:kworker.0 Request processed in 61us (722us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319775.457362:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4821 00000100:00080000:0.0:1767319775.457383:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319775.457413:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985254912:0@lo:400:kworker.0 00000100:00080000:0.0:1767319775.457444:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319775.457448:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319775.457451:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985254528:0@lo:38:kworker.0 00000100:00100000:1.0:1767319775.462637:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985254656:12345-0@lo:400:kworker.0 Request processed in 9008us (9214us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319775.462654:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4820 00000100:00100000:3.0:1767319775.462888:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985254656:0@lo:400:kworker.0 00000100:00080000:2.0:1767319775.463252:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319775.463258:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319775.463263:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319775.463270:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5018 00000100:00080000:2.0:1767319775.463275:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767319775.466425:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319775.466471:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98a80 x1853163985255040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319775.466659:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985255040:0@lo:38:kworker.0 00000100:00100000:0.0:1767319775.466723:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319775.466798:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255040 00000100:00100000:1.0:1767319775.466806:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255040 00000100:00100000:2.0:1767319775.466829:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319775.466843:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985255168:0@lo:400:kworker.0 00000100:00100000:2.0:1767319775.466876:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319775.466916:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255168 00000100:00100000:2.0:1767319775.466920:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255168 00000100:00100000:2.0:1767319775.466980:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3153 00000100:00100000:1.0:1767319775.466987:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5365 00000100:00100000:2.0:1767319775.466992:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985255168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319775.467010:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985255040:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319775.467033:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319775.467049:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985255168:12345-0@lo:400:kworker.0 Request processed in 60us (170us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319775.467057:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3153 00000100:00080000:2.0:1767319775.467079:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319775.467107:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985255168:0@lo:400:kworker.0 00000100:00100000:2.0:1767319775.467143:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985255296:0@lo:400:kworker.0 00000100:00100000:2.0:1767319775.468758:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319775.468767:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985255040:12345-0@lo:38:kworker.0 Request processed in 1769us (2042us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319775.468778:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5365 00000100:00080000:0.0:1767319775.468884:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319775.468888:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319775.468891:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985255040:0@lo:38:kworker.0 00000100:00100000:3.0:1767319775.469020:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255296 00000100:00100000:3.0:1767319775.469025:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255296 00000100:00100000:3.0:1767319775.469113:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3154 00000100:00100000:3.0:1767319775.469123:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985255296:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319775.469126:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319775.469206:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985255296:12345-0@lo:400:kworker.0 Request processed in 84us (454us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319775.469212:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3154 00000100:00100000:1.0:1767319775.469267:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985255296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319775.963444:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985255552:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319775.963520:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319775.963568:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319775.963575:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985255424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319775.963597:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319775.963931:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255552 00000100:00100000:2.0:1767319775.963935:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255424 00000100:00100000:0.0:1767319775.963939:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255552 00000100:00100000:2.0:1767319775.963939:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255424 00000100:00100000:2.0:1767319775.964031:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2336 00000100:00100000:2.0:1767319775.964040:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985255424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319775.964081:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319775.964141:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985255424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 102us (542us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319775.964149:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2336 00000100:00100000:2.0:1767319775.964193:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2337 00000100:00100000:2.0:1767319775.964200:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985255552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319775.964214:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319775.964239:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985255552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 40us (720us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319775.964245:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2337 00000100:00100000:3.0:1767319775.964374:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985255424:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319775.964388:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985255552:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319776.091357:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319776.426903:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319776.426979:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319776.427970:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920579584 00000100:00100000:3.0:1767319776.427982:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920579584 00000100:00100000:3.0:1767319776.428068:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920579712 00000100:00100000:3.0:1767319776.428071:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920579712 00000100:00100000:3.0:1767319776.428131:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4822 00000100:00100000:3.0:1767319776.428140:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920579712:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319776.428222:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920579712:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (1243us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319776.428234:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4822 00000100:00100000:3.0:1767319776.428251:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5366 00000100:00100000:3.0:1767319776.428255:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920579584:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319776.428274:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319776.428293:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920579584:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 37us (1401us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319776.428299:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5366 00000100:00080000:2.0:1767319776.430885:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319777.115432:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319778.139738:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319779.163443:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319780.187357:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319780.571377:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319780.571398:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319780.571401:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319780.571405:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319780.571409:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319780.571418:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5023 00000100:00080000:3.0:1767319780.571422:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319780.571430:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319780.571471:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985255680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319780.571502:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319780.571505:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319780.571507:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319780.571511:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5023 00000100:00080000:3.0:1767319780.571513:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319780.571516:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319780.571532:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633800 x1853163985255808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319780.571620:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319780.571668:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985255680:0@lo:38:kworker.0 00000100:00100000:2.0:1767319780.571739:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319780.571794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985255808:0@lo:38:kworker.0 00000100:00100000:2.0:1767319780.571821:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319780.571857:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255680 00000100:00100000:2.0:1767319780.571865:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255680 00000100:00100000:2.0:1767319780.571900:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5367 00000100:00100000:2.0:1767319780.571907:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985255680:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319780.571928:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319780.571968:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985255680:12345-0@lo:38:kworker.0 Request processed in 60us (228us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319780.571974:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5367 00000100:00080000:2.0:1767319780.572015:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319780.572020:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319780.572023:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985255680:0@lo:38:kworker.0 00000100:00100000:0.0:1767319780.572160:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255808 00000100:00100000:0.0:1767319780.572195:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255808 00000100:00100000:0.0:1767319780.572218:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1832 00000100:00100000:0.0:1767319780.572223:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985255808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319780.572235:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319780.572267:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985255808:12345-0@lo:38:kworker.0 Request processed in 43us (445us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319780.572273:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1832 00000100:00080000:3.0:1767319780.572342:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319780.572346:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319780.572348:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319780.572352:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5023 00000100:00080000:3.0:1767319780.572355:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319780.572358:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319780.572377:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632d80 x1853163985256064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319780.572446:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985256064:0@lo:38:kworker.0 00000100:00100000:2.0:1767319780.572472:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319780.572502:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319780.572506:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319780.572509:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985255808:0@lo:38:kworker.0 00000100:00100000:0.0:1767319780.572608:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256064 00000100:00100000:0.0:1767319780.572612:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256064 00000100:00100000:0.0:1767319780.572641:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5368 00000100:00100000:0.0:1767319780.572646:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985256064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319780.572657:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319780.572686:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985256064:12345-0@lo:38:kworker.0 Request processed in 39us (213us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319780.572693:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5368 00000100:00080000:2.0:1767319780.573193:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319780.573198:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319780.573201:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985256064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319780.574196:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985256192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319780.574254:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319780.574293:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985255936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319780.574314:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319780.574428:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256192 00000100:00100000:0.0:1767319780.574435:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256192 00000100:00100000:0.0:1767319780.574495:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985255936 00000100:00100000:0.0:1767319780.574499:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985255936 00000100:00100000:0.0:1767319780.574525:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4823 00000100:00100000:0.0:1767319780.574533:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985256192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319780.574586:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985256192:12345-0@lo:400:kworker.0 Request processed in 54us (332us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319780.574593:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4823 00000100:00100000:0.0:1767319780.574610:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4824 00000100:00100000:0.0:1767319780.574616:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985255936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319780.574643:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985255936:12345-0@lo:400:kworker.0 Request processed in 28us (328us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319780.574649:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4824 00000100:00080000:1.0:1767319780.576926:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319780.577001:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985256192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319780.577031:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985255936:0@lo:400:kworker.0 00000100:00080000:3.0:1767319780.585416:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319780.585428:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319780.585433:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319780.585441:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5023 00000100:00080000:3.0:1767319780.585446:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319780.585453:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319780.585493:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631f80 x1853163985256320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319780.585692:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985256448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319780.585758:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319780.585805:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319780.585811:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985256576:0@lo:400:kworker.0 00000100:00100000:0.0:1767319780.585832:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319780.585864:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256448 00000100:00100000:0.0:1767319780.585871:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256448 00000100:00100000:0.0:1767319780.585927:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256576 00000100:00100000:0.0:1767319780.585930:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256576 00000100:00100000:0.0:1767319780.585951:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3155 00000100:00100000:0.0:1767319780.585959:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985256448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319780.586007:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985256448:12345-0@lo:400:kworker.0 Request processed in 48us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319780.586014:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3155 00000100:00100000:0.0:1767319780.586045:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3156 00000100:00100000:0.0:1767319780.586052:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985256576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319780.586079:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985256576:12345-0@lo:400:kworker.0 Request processed in 28us (245us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319780.586087:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3156 00000100:00100000:0.0:1767319780.586129:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985256576:0@lo:400:kworker.0 00000100:00100000:0.0:1767319780.586140:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985256448:0@lo:400:kworker.0 00000100:00100000:2.0:1767319780.586237:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985256320:0@lo:38:kworker.0 00000100:00100000:2.0:1767319780.586269:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319780.586361:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319780.586366:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319780.586751:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256320 00000100:00100000:0.0:1767319780.586757:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256320 00000100:00100000:0.0:1767319780.586793:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5369 00000100:00100000:0.0:1767319780.586798:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985256320:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319780.586817:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319780.586864:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985256320:12345-0@lo:38:kworker.0 Request processed in 64us (593us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319780.586870:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5369 00000100:00080000:2.0:1767319780.586943:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319780.586949:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319780.586952:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985256320:0@lo:38:kworker.0 00000100:00100000:3.0:1767319781.084424:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985256704:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319781.084512:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319781.084611:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256704 00000100:00100000:1.0:1767319781.084620:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256704 00000100:00100000:1.0:1767319781.084670:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2338 00000100:00100000:1.0:1767319781.084679:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985256704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319781.084707:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319781.088070:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985256832:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319781.088137:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319781.088211:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256832 00000100:00100000:2.0:1767319781.088219:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256832 00000100:00100000:2.0:1767319781.088273:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2339 00000100:00100000:2.0:1767319781.088282:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985256832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319781.088312:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319781.088356:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985256832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (218us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319781.088364:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2339 00000100:00100000:2.0:1767319781.088420:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985256832:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319781.094882:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985256704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 10193us (10358us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319781.094900:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2338 00000100:00100000:3.0:1767319781.096017:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985256704:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319781.213503:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319781.547592:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319781.547713:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920579840 00000100:00100000:3.0:1767319781.547727:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920579840 00000100:00100000:3.0:1767319781.547778:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5370 00000100:00100000:3.0:1767319781.547786:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920579840:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319781.547942:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319781.548226:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920579840:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 437us (644us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319781.548234:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5370 00000100:00100000:3.0:1767319781.566427:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319781.566567:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920579968 00000100:00100000:0.0:1767319781.566586:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920579968 00000100:00080000:1.0:1767319781.566897:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319781.566962:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4825 00000100:00100000:0.0:1767319781.566974:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920579968:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319781.567379:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920579968:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 395us (956us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319781.567389:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4825 00010000:00010000:1.0:1767319782.240696:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319783.265920:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319784.288688:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319785.307991:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319785.691328:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319785.691338:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319785.691342:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319785.691345:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319785.691350:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319785.691358:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5028 00000100:00080000:3.0:1767319785.691362:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319785.691369:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319785.691417:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985256960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319785.691441:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319785.691444:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319785.691446:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319785.691449:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5028 00000100:00080000:3.0:1767319785.691451:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319785.691455:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319785.691469:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631f80 x1853163985257088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319785.691506:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319785.691510:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319785.691512:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319785.691515:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5028 00000100:00080000:3.0:1767319785.691517:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319785.691520:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319785.691530:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630000 x1853163985257344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319785.691545:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319785.691548:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319785.691549:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319785.691552:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5028 00000100:00080000:3.0:1767319785.691554:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319785.691557:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319785.691565:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6300 x1853163985257600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319785.691635:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985257472:0@lo:400:kworker.0 00000100:00100000:3.0:1767319785.691692:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319785.691750:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319785.691756:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985257216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319785.691778:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319785.691810:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985257728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319785.691838:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319785.691855:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319785.691860:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985257856:0@lo:400:kworker.0 00000100:00100000:3.0:1767319785.691881:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319785.691912:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257472 00000100:00100000:3.0:1767319785.691919:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257472 00000100:00100000:3.0:1767319785.691966:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257216 00000100:00100000:3.0:1767319785.691970:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257216 00000100:00100000:3.0:1767319785.691991:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4826 00000100:00100000:3.0:1767319785.691999:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985257472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319785.692048:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985257472:12345-0@lo:400:kworker.0 Request processed in 49us (355us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319785.692055:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4826 00000100:00100000:3.0:1767319785.692073:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4827 00000100:00100000:3.0:1767319785.692079:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985257216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319785.692105:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985257216:12345-0@lo:400:kworker.0 Request processed in 26us (325us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319785.692110:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4827 00000100:00080000:3.0:1767319785.692126:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319785.692155:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985257216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319785.692191:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985257472:0@lo:400:kworker.0 00000100:00100000:0.0:1767319785.693745:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985256960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319785.693806:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319785.693850:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985257088:0@lo:38:kworker.0 00000100:00100000:0.0:1767319785.693875:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319785.693903:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985257344:0@lo:38:kworker.0 00000100:00100000:0.0:1767319785.693924:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319785.693941:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985257600:0@lo:38:kworker.0 00000100:00100000:0.0:1767319785.693962:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319785.694000:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985256960 00000100:00100000:0.0:1767319785.694006:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985256960 00000100:00100000:0.0:1767319785.694028:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257344 00000100:00100000:0.0:1767319785.694031:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257344 00000100:00100000:0.0:1767319785.694039:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257600 00000100:00100000:0.0:1767319785.694041:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257600 00000100:00100000:0.0:1767319785.694052:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5371 00000100:00100000:0.0:1767319785.694058:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985256960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319785.694078:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319785.694114:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985256960:12345-0@lo:38:kworker.0 Request processed in 55us (308us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319785.694121:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5371 00000100:00100000:0.0:1767319785.694137:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5372 00000100:00100000:0.0:1767319785.694141:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985257344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319785.694150:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319785.694191:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985257344:12345-0@lo:38:kworker.0 Request processed in 48us (266us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319785.694197:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5372 00000100:00100000:0.0:1767319785.694209:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5373 00000100:00100000:0.0:1767319785.694213:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985257600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319785.694222:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319785.694256:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985257600:12345-0@lo:38:kworker.0 Request processed in 41us (293us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319785.694264:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5373 00000100:00100000:0.0:1767319785.694284:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257728 00000100:00100000:0.0:1767319785.694288:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257728 00000100:00100000:0.0:1767319785.694339:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257856 00000100:00100000:0.0:1767319785.694342:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257856 00000100:00100000:0.0:1767319785.694362:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3157 00000100:00100000:0.0:1767319785.694368:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a06850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985257728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319785.694506:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257088 00000100:00100000:2.0:1767319785.694510:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257088 00000100:00100000:2.0:1767319785.694547:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1833 00000100:00100000:2.0:1767319785.694553:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985257088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319785.694563:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319785.694669:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3158 00000100:00100000:3.0:1767319785.694678:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985257856:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319785.694918:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319785.694922:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319785.694953:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319785.694958:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319785.694962:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985256960:0@lo:38:kworker.0 00000100:00080000:1.0:1767319785.694972:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319785.695020:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319785.695023:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985257344:0@lo:38:kworker.0 00000100:00080000:1.0:1767319785.695034:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319785.695037:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319785.695040:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985257600:0@lo:38:kworker.0 00000100:00100000:2.0:1767319785.697646:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985257088:12345-0@lo:38:kworker.0 Request processed in 3083us (3761us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319785.697661:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1833 00000100:00080000:1.0:1767319785.697809:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319785.697815:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319785.697818:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985257088:0@lo:38:kworker.0 00000100:00100000:3.0:1767319785.698468:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985257856:12345-0@lo:400:kworker.0 Request processed in 3789us (6582us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319785.698479:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3158 00000100:00100000:2.0:1767319785.698895:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985257856:0@lo:400:kworker.0 00000100:00100000:0.0:1767319785.701924:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a06850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985257728:12345-0@lo:400:kworker.0 Request processed in 7546us (10074us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319785.701939:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3157 00000100:00100000:2.0:1767319785.702215:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985257728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319786.203441:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1767319786.203465:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985257984:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319786.203531:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319786.203570:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985258112:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319786.203593:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319786.203620:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985257984 00000100:00100000:1.0:1767319786.203629:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985257984 00000100:00100000:2.0:1767319786.203661:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985258112 00000100:00100000:2.0:1767319786.203665:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985258112 00000100:00100000:1.0:1767319786.203689:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2340 00000100:00100000:1.0:1767319786.203743:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985257984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319786.203772:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319786.203817:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985257984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (287us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319786.203840:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2341 00000100:00100000:3.0:1767319786.203888:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985257984:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319786.204010:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985258112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319786.204030:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319786.204119:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985258112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 271us (524us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319786.204667:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2341 00000100:00100000:3.0:1767319786.204745:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985258112:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319786.216100:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2340 00010000:00010000:1.0:1767319786.331549:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319786.667210:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319786.667245:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319786.667324:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920580096 00000100:00100000:1.0:1767319786.667334:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920580096 00000100:00100000:1.0:1767319786.667380:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920580224 00000100:00100000:1.0:1767319786.667383:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920580224 00000100:00100000:1.0:1767319786.667440:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4828 00000100:00100000:1.0:1767319786.667449:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920580224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319786.667521:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920580224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 73us (274us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319786.667528:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4828 00000100:00080000:2.0:1767319786.667543:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319786.667546:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5374 00000100:00100000:1.0:1767319786.667551:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920580096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319786.667571:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319786.667602:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920580096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (444us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319786.667607:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5374 00010000:00010000:1.0:1767319787.357742:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319788.379404:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319789.403882:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319790.427818:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319790.764746:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319790.764781:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920580352 offset 0 length 520: 4 00000100:00100000:3.0:1767319790.764799:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319790.764882:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920580608 00000100:00100000:0.0:1767319790.764891:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920580608 00000100:00100000:0.0:1767319790.764940:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920580480 00000100:00100000:0.0:1767319790.764943:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920580480 00000100:00100000:0.0:1767319790.764957:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4829 00000100:00100000:0.0:1767319790.764968:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920580608:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319790.764978:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319790.765016:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5375 00000100:00100000:3.0:1767319790.765024:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920580480:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:0.0:1767319790.765050:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920580608:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (309us total) trans 0 rc 0/0 00010000:00020000:3.0:1767319790.765059:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319790.765060:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4829 00000100:00100000:3.0:1767319790.765093:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920580480:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 68us (292us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319790.765100:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5375 00000100:00080000:3.0:1767319790.811300:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.811310:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319790.811314:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319790.811317:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319765/real 1767319765] req@ffff8ce188630380 x1853163985251712/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319790 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319790.811319:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.811324:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.811333:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5033 00000100:00080000:3.0:1767319790.811337:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.811344:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319790.811353:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188630380 x1853163985251712/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319790 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319790.811383:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319790.811384:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ea00 x1853163985258240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319790.811403:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00080000:3.0:1767319790.811409:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319790.811410:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985251712:0@lo:250:kworker.0 00000100:00080000:3.0:1767319790.811416:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.811420:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.811424:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5033 00000100:00080000:3.0:1767319790.811426:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.811430:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319790.811443:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985258240:0@lo:38:kworker.0 00000100:00100000:3.0:1767319790.811447:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985258368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319790.811480:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.811496:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.811498:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.811502:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5033 00000100:00080000:3.0:1767319790.811505:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.811509:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319790.811516:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319790.811523:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c700 x1853163985258624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319790.811549:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.811553:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.811555:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.811559:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5033 00000100:00080000:3.0:1767319790.811561:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.811564:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319790.811573:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985258368:0@lo:38:kworker.0 00000100:00100000:3.0:1767319790.811577:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612df80 x1853163985258880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319790.811607:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319790.811609:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.811614:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319790.811616:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767319790.811633:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985258624:0@lo:38:kworker.0 00000100:00100000:0.0:1767319790.811650:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319790.811665:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985258880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319790.811681:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319790.811737:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985259008:0@lo:400:kworker.0 00000100:00100000:0.0:1767319790.811763:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319790.811783:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319790.811788:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985258752:0@lo:400:kworker.0 00000100:00100000:0.0:1767319790.811808:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319790.811812:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5013 00000100:00080000:3.0:1767319790.811816:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.811821:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319790.811835:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d180 x1853163985259264/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319790.811838:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985259008 00000100:00100000:0.0:1767319790.811848:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985259008 00000100:00100000:3.0:1767319790.811871:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985258240 00000100:00100000:3.0:1767319790.811877:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985258240 00000100:00100000:0.0:1767319790.811902:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3159 00000100:00100000:0.0:1767319790.811911:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985259008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319790.811959:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985259008:12345-0@lo:400:kworker.0 Request processed in 49us (194us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319790.811966:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3159 00000100:00100000:0.0:1767319790.812002:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985259008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319790.812020:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985258624 00000100:00100000:0.0:1767319790.812026:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985258880 00000100:00100000:3.0:1767319790.812026:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985258624 00000100:00100000:0.0:1767319790.812033:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985258880 00000100:00100000:3.0:1767319790.812058:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985258752 00000100:00100000:3.0:1767319790.812063:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985258752 00000100:00100000:0.0:1767319790.812069:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5376 00000100:00100000:0.0:1767319790.812086:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985258240:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319790.812093:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4830 00000100:00100000:3.0:1767319790.812103:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985258752:12345-0@lo:400:kworker.0 00010000:00020000:0.0:1767319790.812126:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319790.812149:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985258752:12345-0@lo:400:kworker.0 Request processed in 46us (339us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319790.812157:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4830 00000100:00100000:3.0:1767319790.812210:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5377 00000100:00100000:3.0:1767319790.812215:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985258880:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319790.812226:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319790.812242:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985258240:12345-0@lo:38:kworker.0 Request processed in 159us (727us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319790.812251:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5376 00000100:00100000:3.0:1767319790.812262:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985258880:12345-0@lo:38:kworker.0 Request processed in 44us (577us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319790.812268:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5377 00000100:00100000:3.0:1767319790.812287:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5378 00000100:00100000:3.0:1767319790.812292:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985258624:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319790.812301:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319790.812323:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985258624:12345-0@lo:38:kworker.0 Request processed in 30us (672us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319790.812397:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985259136:0@lo:400:kworker.0 00000100:00100000:1.0:1767319790.812433:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319790.812457:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319790.812462:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985258496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319790.812481:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319790.812503:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985258496 00000100:00100000:1.0:1767319790.812506:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985258496 00000100:00100000:2.0:1767319790.812579:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985258368 00000100:00100000:2.0:1767319790.812585:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985258368 00000100:00100000:2.0:1767319790.812609:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1834 00000100:00100000:2.0:1767319790.812614:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985258368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319790.812624:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319790.812652:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985258368:12345-0@lo:38:kworker.0 Request processed in 38us (1046us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319790.812658:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1834 00000100:00080000:2.0:1767319790.812760:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319790.812765:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319790.812774:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985259136 00000100:00100000:2.0:1767319790.812777:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985259136 00000100:00100000:2.0:1767319790.812806:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3160 00000100:00100000:2.0:1767319790.812812:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985259136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319790.812845:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985259136:12345-0@lo:400:kworker.0 Request processed in 34us (412us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319790.812852:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3160 00000100:00080000:2.0:1767319790.812866:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319790.812875:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985259264:0@lo:250:kworker.0 00000400:00000100:2.0:1767319790.812895:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985259264 offset 0 length 520: 4 00000100:00080000:2.0:1767319790.812919:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319790.812923:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319790.812926:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985258880:0@lo:38:kworker.0 00000100:00080000:2.0:1767319790.812934:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319790.812937:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319790.812939:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985258368:0@lo:38:kworker.0 00000100:00080000:2.0:1767319790.812946:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319790.812949:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319790.812951:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985258624:0@lo:38:kworker.0 00000100:00080000:2.0:1767319790.812957:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319790.812960:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319790.812962:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985258240:0@lo:38:kworker.0 00000100:00100000:2.0:1767319790.813005:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985258752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319790.813050:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985259136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319790.815027:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5378 00000100:00100000:1.0:1767319790.815643:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4831 00000100:00100000:1.0:1767319790.815662:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985258496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319790.815752:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985258496:12345-0@lo:400:kworker.0 Request processed in 99us (3269us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319790.815924:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4831 00000100:00100000:2.0:1767319790.815947:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985258496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319791.323727:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985259392:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319791.323797:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319791.323893:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985259392 00000100:00100000:1.0:1767319791.323901:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985259392 00000100:00100000:1.0:1767319791.323958:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2342 00000100:00100000:1.0:1767319791.323966:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985259392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319791.323995:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319791.324298:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985259392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 331us (500us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319791.324306:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2342 00000100:00100000:3.0:1767319791.324489:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985259392:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319791.324623:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985259520:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319791.324653:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319791.324746:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985259520 00000100:00100000:2.0:1767319791.324752:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985259520 00000100:00100000:2.0:1767319791.324782:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2343 00000100:00100000:2.0:1767319791.324788:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985259520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319791.324803:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319791.324842:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985259520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (188us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319791.324848:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2343 00000100:00100000:3.0:1767319791.325079:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985259520:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319791.451652:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319792.475371:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319793.499364:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319794.526433:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319795.551016:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319795.882749:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319795.882805:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319795.882918:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920580864 00000100:00100000:2.0:1767319795.882943:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920580864 00000100:00100000:2.0:1767319795.883019:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920580736 00000100:00100000:2.0:1767319795.883022:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920580736 00000100:00100000:2.0:1767319795.883046:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4832 00000100:00100000:2.0:1767319795.883057:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920580864:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319795.883145:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920580864:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (405us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319795.883222:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4832 00000100:00100000:1.0:1767319795.883253:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5379 00000100:00100000:1.0:1767319795.883266:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920580736:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319795.883297:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319795.883364:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920580736:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 98us (559us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767319795.883772:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319795.886843:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5379 00000100:00080000:2.0:1767319795.931406:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.931412:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319795.931422:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.931426:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.931431:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.931438:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5039 00000100:00080000:2.0:1767319795.931443:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.931449:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319795.931486:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163985259648/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319795.931502:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.931505:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.931507:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.931510:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5039 00000100:00080000:2.0:1767319795.931513:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.931515:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319795.931524:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77480 x1853163985259776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319795.931552:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.931555:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.931557:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.931561:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5039 00000100:00080000:2.0:1767319795.931563:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.931565:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319795.931573:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75880 x1853163985260032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319795.931591:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.931594:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.931595:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.931598:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5039 00000100:00080000:2.0:1767319795.931600:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.931603:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319795.931610:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163985260288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319795.931758:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985260544:0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.931814:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319795.931864:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319795.931870:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985259904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.931889:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319795.931916:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260544 00000100:00100000:2.0:1767319795.931922:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260544 00000100:00100000:2.0:1767319795.931966:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3161 00000100:00100000:2.0:1767319795.931974:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985260544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.932014:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985260544:12345-0@lo:400:kworker.0 Request processed in 41us (200us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319795.932021:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3161 00000100:00100000:2.0:1767319795.932075:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985260544:0@lo:400:kworker.0 00000100:00100000:3.0:1767319795.932183:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985260416:0@lo:400:kworker.0 00000100:00100000:3.0:1767319795.932211:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319795.932236:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319795.932241:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985260160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319795.932261:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319795.932286:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985259904 00000100:00100000:3.0:1767319795.932290:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985259904 00000100:00100000:3.0:1767319795.932318:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260160 00000100:00100000:3.0:1767319795.932320:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260160 00000100:00100000:3.0:1767319795.932337:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4833 00000100:00100000:3.0:1767319795.932343:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985259904:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319795.932377:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985259904:12345-0@lo:400:kworker.0 Request processed in 33us (486us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319795.932382:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4833 00000100:00100000:3.0:1767319795.932394:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4834 00000100:00100000:3.0:1767319795.932399:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985260160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319795.932421:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985260160:12345-0@lo:400:kworker.0 Request processed in 22us (159us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319795.932426:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4834 00000100:00100000:3.0:1767319795.932452:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985260160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319795.932468:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985259904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.932974:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260416 00000100:00100000:2.0:1767319795.932979:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260416 00000100:00100000:0.0:1767319795.933094:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985259648:0@lo:38:kworker.0 00000100:00100000:0.0:1767319795.933139:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319795.933205:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985259776:0@lo:38:kworker.0 00000100:00100000:0.0:1767319795.933229:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319795.933254:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985260032:0@lo:38:kworker.0 00000100:00100000:0.0:1767319795.933273:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319795.933289:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985260288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319795.933307:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319795.933330:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319795.933334:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319795.933336:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319795.933346:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985259776 00000100:00100000:0.0:1767319795.933349:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985259776 00000100:00100000:0.0:1767319795.933369:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1835 00000100:00100000:0.0:1767319795.933374:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985259776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319795.933395:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319795.933438:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985259776:12345-0@lo:38:kworker.0 Request processed in 62us (207us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319795.933444:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1835 00000100:00100000:0.0:1767319795.933461:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985259648 00000100:00100000:0.0:1767319795.933464:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985259648 00000100:00100000:0.0:1767319795.933478:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260032 00000100:00100000:0.0:1767319795.933480:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260032 00000100:00100000:0.0:1767319795.933490:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260288 00000100:00100000:0.0:1767319795.933492:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260288 00000100:00100000:0.0:1767319795.933504:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5380 00000100:00100000:0.0:1767319795.933507:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985259648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319795.933516:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319795.933536:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985259648:12345-0@lo:38:kworker.0 Request processed in 27us (397us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319795.933541:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5380 00000100:00100000:0.0:1767319795.933550:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5381 00000100:00100000:0.0:1767319795.933553:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985260032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319795.933561:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319795.933575:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985260032:12345-0@lo:38:kworker.0 Request processed in 20us (301us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319795.933580:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5381 00000100:00100000:0.0:1767319795.933587:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5382 00000100:00100000:0.0:1767319795.933591:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985260288:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319795.933598:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319795.933611:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985260288:12345-0@lo:38:kworker.0 Request processed in 19us (303us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319795.933616:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5382 00000100:00080000:0.0:1767319795.933645:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319795.933649:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319795.933653:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985259648:0@lo:38:kworker.0 00000100:00080000:0.0:1767319795.933660:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319795.933663:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319795.933666:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985259776:0@lo:38:kworker.0 00000100:00080000:0.0:1767319795.933672:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319795.933675:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319795.933678:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985260032:0@lo:38:kworker.0 00000100:00080000:0.0:1767319795.933683:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319795.933686:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319795.933688:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985260288:0@lo:38:kworker.0 00000100:00100000:2.0:1767319795.933791:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3162 00000100:00100000:2.0:1767319795.933800:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985260416:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.933845:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985260416:12345-0@lo:400:kworker.0 Request processed in 48us (1632us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319795.933850:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3162 00000100:00100000:0.0:1767319795.933906:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985260416:0@lo:400:kworker.0 00000100:00100000:3.0:1767319796.443886:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319796.443913:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985260672:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319796.443986:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319796.444235:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319796.444245:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985260800:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319796.444279:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319796.444402:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260672 00000100:00100000:2.0:1767319796.444411:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260672 00000100:00100000:2.0:1767319796.444461:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260800 00000100:00100000:2.0:1767319796.444465:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260800 00000100:00100000:2.0:1767319796.444496:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2344 00000100:00100000:2.0:1767319796.444505:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985260672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319796.444535:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319796.444585:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985260672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (598us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319796.444593:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2344 00000100:00100000:2.0:1767319796.444611:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2345 00000100:00100000:2.0:1767319796.444618:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb6c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985260800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319796.444633:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319796.444665:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb6c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985260800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (385us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319796.444671:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2345 00000100:00100000:1.0:1767319796.444871:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985260672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319796.444908:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985260800:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319796.571413:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319797.596878:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319798.619752:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319799.646287:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319800.667317:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319801.004612:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319801.004655:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319801.004734:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920580992 00000100:00100000:1.0:1767319801.004744:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920580992 00000100:00100000:1.0:1767319801.004803:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920581120 00000100:00100000:1.0:1767319801.004807:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920581120 00000100:00100000:1.0:1767319801.004861:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4835 00000100:00100000:1.0:1767319801.004871:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920581120:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319801.004942:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920581120:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 72us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319801.004950:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4835 00000100:00100000:1.0:1767319801.004980:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5383 00000100:00100000:1.0:1767319801.004985:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920580992:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319801.005004:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319801.005040:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920580992:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (440us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319801.005047:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5383 00000100:00080000:3.0:1767319801.005105:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319801.051656:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319801.051666:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319801.051669:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319801.051673:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319801.051678:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319801.051686:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5044 00000100:00080000:2.0:1767319801.051691:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319801.051697:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319801.051737:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77100 x1853163985260928/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319801.051774:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319801.051778:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319801.051780:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319801.051784:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5044 00000100:00080000:2.0:1767319801.051786:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319801.051790:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319801.051803:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985261056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319801.051853:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319801.051857:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319801.051859:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319801.051863:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5044 00000100:00080000:2.0:1767319801.051866:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319801.051869:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319801.051882:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985261312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319801.051911:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319801.051914:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319801.051916:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319801.051919:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5044 00000100:00080000:2.0:1767319801.051921:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319801.051924:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319801.051935:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163985261568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319801.051973:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985261696:0@lo:400:kworker.0 00000100:00100000:0.0:1767319801.052084:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985260928:0@lo:38:kworker.0 00000100:00100000:0.0:1767319801.052138:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319801.052216:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985261056:0@lo:38:kworker.0 00000100:00100000:0.0:1767319801.052241:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319801.052264:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985261312:0@lo:38:kworker.0 00000100:00100000:0.0:1767319801.052284:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319801.052316:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985261568:0@lo:38:kworker.0 00000100:00100000:0.0:1767319801.052336:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319801.052373:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261056 00000100:00100000:0.0:1767319801.052380:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261056 00000100:00100000:0.0:1767319801.052414:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1836 00000100:00100000:0.0:1767319801.052420:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985261056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319801.052439:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319801.052473:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985261056:12345-0@lo:38:kworker.0 Request processed in 52us (231us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319801.052479:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1836 00000100:00100000:0.0:1767319801.052503:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985260928 00000100:00100000:0.0:1767319801.052506:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985260928 00000100:00100000:0.0:1767319801.052520:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261312 00000100:00100000:0.0:1767319801.052522:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261312 00000100:00100000:0.0:1767319801.052532:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261568 00000100:00100000:0.0:1767319801.052534:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261568 00000100:00100000:0.0:1767319801.052547:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5384 00000100:00100000:0.0:1767319801.052551:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985260928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319801.052560:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319801.052585:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985260928:12345-0@lo:38:kworker.0 Request processed in 33us (447us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319801.052591:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5384 00000100:00100000:0.0:1767319801.052602:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5385 00000100:00100000:0.0:1767319801.052605:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985261312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319801.052613:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319801.052634:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985261312:12345-0@lo:38:kworker.0 Request processed in 27us (349us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319801.052639:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5385 00000100:00100000:0.0:1767319801.052650:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5386 00000100:00100000:0.0:1767319801.052654:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985261568:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319801.052662:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319801.052683:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985261568:12345-0@lo:38:kworker.0 Request processed in 28us (346us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319801.052689:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5386 00000100:00080000:0.0:1767319801.052727:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319801.052732:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319801.052735:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985260928:0@lo:38:kworker.0 00000100:00080000:0.0:1767319801.052742:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319801.052745:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319801.052747:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985261056:0@lo:38:kworker.0 00000100:00080000:0.0:1767319801.052754:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319801.052756:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319801.052759:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985261312:0@lo:38:kworker.0 00000100:00080000:0.0:1767319801.052765:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319801.052768:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319801.052770:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985261568:0@lo:38:kworker.0 00000100:00100000:2.0:1767319801.053036:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319801.053067:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319801.053073:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985261440:0@lo:400:kworker.0 00000100:00100000:2.0:1767319801.053095:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319801.053125:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261440 00000100:00100000:2.0:1767319801.053129:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261440 00000100:00100000:2.0:1767319801.053182:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4836 00000100:00100000:2.0:1767319801.053189:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985261440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319801.053226:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985261440:12345-0@lo:400:kworker.0 Request processed in 39us (130us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319801.053233:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4836 00000100:00100000:2.0:1767319801.053262:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985261440:0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053450:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985261184:0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053479:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319801.053503:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319801.053508:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985261824:0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053526:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319801.053552:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261696 00000100:00100000:1.0:1767319801.053556:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261696 00000100:00100000:1.0:1767319801.053585:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261824 00000100:00100000:1.0:1767319801.053589:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261824 00000100:00100000:1.0:1767319801.053609:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3163 00000100:00100000:1.0:1767319801.053615:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985261696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053651:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985261696:12345-0@lo:400:kworker.0 Request processed in 36us (615us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319801.053657:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3163 00000100:00100000:1.0:1767319801.053670:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3164 00000100:00100000:1.0:1767319801.053675:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985261824:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053704:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985261824:12345-0@lo:400:kworker.0 Request processed in 29us (176us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319801.053710:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3164 00000100:00100000:1.0:1767319801.053744:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985261824:0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053760:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261184 00000100:00100000:1.0:1767319801.053763:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261184 00000100:00100000:1.0:1767319801.053782:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4837 00000100:00100000:1.0:1767319801.053788:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985261184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053820:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985261184:12345-0@lo:400:kworker.0 Request processed in 31us (340us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319801.053826:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4837 00000100:00100000:1.0:1767319801.053847:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985261184:0@lo:400:kworker.0 00000100:00100000:1.0:1767319801.053866:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985261696:0@lo:400:kworker.0 00000100:00080000:3.0:1767319801.054083:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319801.054088:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319801.054090:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319801.565273:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319801.565295:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985261952:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319801.565365:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319801.565538:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985261952 00000100:00100000:0.0:1767319801.565547:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985261952 00000100:00100000:0.0:1767319801.565614:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2346 00000100:00100000:0.0:1767319801.565624:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985261952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319801.565657:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319801.565705:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985261952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319801.565713:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2346 00000100:00100000:0.0:1767319801.565760:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985261952:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319801.566231:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985262080:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319801.566267:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319801.566413:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262080 00000100:00100000:3.0:1767319801.566419:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262080 00000100:00100000:3.0:1767319801.566460:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2347 00000100:00100000:3.0:1767319801.566468:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985262080:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319801.566486:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319801.566543:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985262080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (274us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319801.566550:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2347 00000100:00100000:1.0:1767319801.566777:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985262080:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319801.692342:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319802.715473:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319803.741270:0:71910:0:(ldlm_pool.c:1331:ldlm_pools_recalc_task()) Negative interval(0) 00010000:00010000:1.0:1767319803.743143:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319804.764141:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319805.787470:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319806.124242:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319806.124302:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319806.124455:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920581376 00000100:00100000:0.0:1767319806.124481:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920581376 00000100:00100000:0.0:1767319806.124566:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920581248 00000100:00100000:0.0:1767319806.124569:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920581248 00000100:00100000:0.0:1767319806.124597:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4838 00000100:00100000:0.0:1767319806.124610:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920581376:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319806.124657:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5387 00000100:00100000:2.0:1767319806.124666:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920581248:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319806.124693:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319806.124924:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319806.125281:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920581376:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 670us (1051us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319806.125291:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4838 00000100:00100000:2.0:1767319806.128959:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920581248:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 4275us (4638us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319806.128977:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5387 00000100:00080000:3.0:1767319806.174350:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319806.174365:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319806.174369:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319806.174374:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319806.174380:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319806.174391:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5049 00000100:00080000:3.0:1767319806.174397:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319806.174407:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319806.174465:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985262208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319806.174503:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319806.174508:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319806.174510:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319806.174514:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5049 00000100:00080000:3.0:1767319806.174516:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319806.174520:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319806.174534:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612df80 x1853163985262336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319806.174591:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319806.174596:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319806.174597:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319806.174602:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5049 00000100:00080000:3.0:1767319806.174605:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319806.174608:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319806.174624:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c380 x1853163985262592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319806.174652:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319806.174655:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319806.174657:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319806.174661:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5049 00000100:00080000:3.0:1767319806.174663:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319806.174666:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319806.174682:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d500 x1853163985262848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319806.175039:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985263104:0@lo:400:kworker.0 00000100:00100000:2.0:1767319806.175146:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319806.175277:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319806.175285:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985262464:0@lo:400:kworker.0 00000100:00100000:2.0:1767319806.175329:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319806.175385:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262464 00000100:00100000:2.0:1767319806.175395:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262464 00000100:00100000:2.0:1767319806.175470:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4839 00000100:00100000:2.0:1767319806.175482:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985262464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319806.175551:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985262464:12345-0@lo:400:kworker.0 Request processed in 71us (221us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319806.175559:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4839 00000100:00080000:2.0:1767319806.175586:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319806.175626:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985262464:0@lo:400:kworker.0 00000100:00100000:3.0:1767319806.176412:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985263104 00000100:00100000:3.0:1767319806.176422:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985263104 00000100:00100000:3.0:1767319806.176492:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3165 00000100:00100000:3.0:1767319806.176501:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985263104:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319806.176563:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985263104:12345-0@lo:400:kworker.0 Request processed in 65us (1418us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319806.176571:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3165 00000100:00100000:0.0:1767319806.176676:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985262720:0@lo:400:kworker.0 00000100:00100000:0.0:1767319806.176733:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319806.176777:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319806.176785:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985262976:0@lo:400:kworker.0 00000100:00100000:0.0:1767319806.176813:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319806.176837:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262976 00000100:00100000:0.0:1767319806.176841:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262976 00000100:00100000:0.0:1767319806.177064:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3166 00000100:00100000:0.0:1767319806.177072:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985262976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319806.177113:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985262976:12345-0@lo:400:kworker.0 Request processed in 42us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319806.177120:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3166 00000100:00100000:0.0:1767319806.177160:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985262976:0@lo:400:kworker.0 00000100:00100000:0.0:1767319806.177231:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262720 00000100:00100000:0.0:1767319806.177235:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262720 00000100:00100000:0.0:1767319806.177506:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4840 00000100:00100000:0.0:1767319806.177514:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985262720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319806.177557:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985262720:12345-0@lo:400:kworker.0 Request processed in 44us (823us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319806.177563:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4840 00000100:00100000:0.0:1767319806.177590:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985262720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319806.177741:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985262208:0@lo:38:kworker.0 00000100:00100000:1.0:1767319806.177783:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319806.177820:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985262336:0@lo:38:kworker.0 00000100:00100000:1.0:1767319806.177844:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319806.177868:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985262592:0@lo:38:kworker.0 00000100:00100000:1.0:1767319806.177893:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319806.177907:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985262848:0@lo:38:kworker.0 00000100:00100000:1.0:1767319806.177941:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319806.177979:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262336 00000100:00100000:1.0:1767319806.177985:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262336 00000100:00100000:1.0:1767319806.178012:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1837 00000100:00100000:1.0:1767319806.178017:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985262336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319806.178040:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319806.178080:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985262336:12345-0@lo:38:kworker.0 Request processed in 62us (235us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319806.178087:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1837 00000100:00100000:1.0:1767319806.178110:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262208 00000100:00100000:1.0:1767319806.178114:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262208 00000100:00100000:1.0:1767319806.178133:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262592 00000100:00100000:1.0:1767319806.178136:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262592 00000100:00100000:1.0:1767319806.178150:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985262848 00000100:00100000:1.0:1767319806.178153:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985262848 00000100:00100000:1.0:1767319806.178190:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5388 00000100:00100000:1.0:1767319806.178195:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985262208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319806.178206:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319806.178232:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985262208:12345-0@lo:38:kworker.0 Request processed in 35us (448us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319806.178238:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5388 00000100:00100000:1.0:1767319806.178249:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5389 00000100:00100000:1.0:1767319806.178252:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985262592:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319806.178261:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319806.178284:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985262592:12345-0@lo:38:kworker.0 Request processed in 30us (390us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319806.178289:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5389 00000100:00100000:1.0:1767319806.178298:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5390 00000100:00100000:1.0:1767319806.178303:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985262848:12345-0@lo:38:kworker.0 00000100:00080000:3.0:1767319806.178561:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319806.178568:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319806.178572:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985262208:0@lo:38:kworker.0 00000100:00080000:3.0:1767319806.178580:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319806.178583:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319806.178587:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985262336:0@lo:38:kworker.0 00000100:00080000:3.0:1767319806.178594:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319806.178597:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319806.178599:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985262592:0@lo:38:kworker.0 00000100:00080000:2.0:1767319806.178925:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319806.178932:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319806.178934:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319806.178968:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985263104:0@lo:400:kworker.0 00010000:00020000:1.0:1767319806.179031:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319806.179084:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985262848:12345-0@lo:38:kworker.0 Request processed in 779us (1141us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319806.179091:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5390 00000100:00080000:3.0:1767319806.179292:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319806.179297:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319806.179301:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985262848:0@lo:38:kworker.0 00000100:00100000:1.0:1767319806.683533:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985263232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319806.683653:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319806.684669:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985263232 00000100:00100000:3.0:1767319806.684682:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985263232 00000100:00100000:3.0:1767319806.684805:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2348 00000100:00100000:3.0:1767319806.684816:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985263232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319806.684858:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319806.684979:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985263232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 164us (1326us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319806.684988:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2348 00000100:00100000:2.0:1767319806.685103:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985263232:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319806.690081:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985263360:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319806.690217:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319806.690412:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985263360 00000100:00100000:0.0:1767319806.690424:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985263360 00000100:00100000:0.0:1767319806.690494:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2349 00000100:00100000:0.0:1767319806.690506:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985263360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319806.690553:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319806.690622:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985263360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 116us (402us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319806.690633:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2349 00000100:00100000:2.0:1767319806.690718:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985263360:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319806.811778:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319807.835672:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319808.862510:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319809.883510:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319810.909001:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319811.242745:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319811.242798:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319811.242974:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920581632 00000100:00100000:1.0:1767319811.242985:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920581632 00000100:00100000:1.0:1767319811.243057:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920581504 00000100:00100000:1.0:1767319811.243061:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920581504 00000100:00100000:1.0:1767319811.243082:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4841 00000100:00100000:1.0:1767319811.243093:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920581632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319811.243222:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319811.243242:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5391 00000100:00100000:3.0:1767319811.243248:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920581504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319811.243278:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319811.243340:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920581504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 91us (541us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319811.243348:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5391 00000100:00100000:1.0:1767319811.243896:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920581632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 804us (1171us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319811.243904:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4841 00000100:00080000:0.0:1767319811.292811:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.292830:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319811.292833:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.292838:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.292842:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.292850:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.292855:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.292862:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319811.292907:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4380 x1853163985263488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319811.292933:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.292937:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.292939:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.292943:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.292945:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.292949:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319811.292962:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4700 x1853163985263616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319811.293005:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.293008:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.293011:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.293014:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.293016:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.293020:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319811.293032:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985263872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319811.293057:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.293061:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.293063:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.293066:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.293069:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.293072:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319811.293083:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7100 x1853163985264128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319811.293303:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985263488:0@lo:38:kworker.0 00000100:00100000:1.0:1767319811.293368:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319811.293438:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985263616:0@lo:38:kworker.0 00000100:00100000:1.0:1767319811.293461:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319811.293484:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985263872:0@lo:38:kworker.0 00000100:00100000:1.0:1767319811.293502:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319811.293517:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985264128:0@lo:38:kworker.0 00000100:00100000:1.0:1767319811.293533:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319811.293565:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985263616 00000100:00100000:1.0:1767319811.293572:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985263616 00000100:00100000:1.0:1767319811.293602:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1838 00000100:00100000:1.0:1767319811.293609:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985263616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319811.293640:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319811.293681:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985263616:12345-0@lo:38:kworker.0 Request processed in 70us (218us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319811.293688:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1838 00000100:00100000:1.0:1767319811.293714:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985263488 00000100:00100000:1.0:1767319811.293718:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985263488 00000100:00100000:1.0:1767319811.293734:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985263872 00000100:00100000:1.0:1767319811.293736:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985263872 00000100:00100000:1.0:1767319811.293746:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985264128 00000100:00100000:1.0:1767319811.293748:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985264128 00000100:00100000:1.0:1767319811.293760:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5392 00000100:00100000:1.0:1767319811.293765:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985263488:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319811.293774:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319811.293797:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985263488:12345-0@lo:38:kworker.0 Request processed in 31us (429us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319811.293803:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5392 00000100:00100000:1.0:1767319811.293813:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5393 00000100:00100000:1.0:1767319811.293828:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985263872:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319811.293837:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319811.293856:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985263872:12345-0@lo:38:kworker.0 Request processed in 27us (354us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319811.293861:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5393 00000100:00100000:1.0:1767319811.293872:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5394 00000100:00100000:1.0:1767319811.293875:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985264128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319811.293884:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319811.293903:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985264128:12345-0@lo:38:kworker.0 Request processed in 26us (369us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319811.293909:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5394 00000100:00080000:1.0:1767319811.293942:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319811.293947:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319811.293950:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985263488:0@lo:38:kworker.0 00000100:00080000:1.0:1767319811.293957:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319811.293960:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319811.293963:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985263616:0@lo:38:kworker.0 00000100:00080000:1.0:1767319811.293969:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319811.293972:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319811.293974:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985263872:0@lo:38:kworker.0 00000100:00080000:1.0:1767319811.293980:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319811.293983:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319811.293985:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985264128:0@lo:38:kworker.0 00000100:00100000:3.0:1767319811.294475:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985263744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319811.294507:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319811.294536:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319811.294542:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985264384:0@lo:400:kworker.0 00000100:00100000:3.0:1767319811.294564:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319811.294593:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985264384 00000100:00100000:3.0:1767319811.294598:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985264384 00000100:00100000:3.0:1767319811.294655:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3167 00000100:00100000:3.0:1767319811.294662:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985264384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319811.294703:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985264384:12345-0@lo:400:kworker.0 Request processed in 41us (137us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319811.294709:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3167 00000100:00080000:3.0:1767319811.294730:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319811.294741:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985263744 00000100:00100000:3.0:1767319811.294744:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985263744 00000100:00100000:3.0:1767319811.294769:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4842 00000100:00100000:3.0:1767319811.294775:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985263744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319811.294799:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985263744:12345-0@lo:400:kworker.0 Request processed in 25us (292us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319811.294804:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4842 00000100:00080000:3.0:1767319811.294815:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319811.294848:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985264384:0@lo:400:kworker.0 00000100:00100000:3.0:1767319811.294856:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985263744:0@lo:400:kworker.0 00000100:00100000:2.0:1767319811.295402:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985264000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319811.295434:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319811.295462:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319811.295468:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985264256:0@lo:400:kworker.0 00000100:00100000:2.0:1767319811.295489:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319811.295519:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985264256 00000100:00100000:2.0:1767319811.295523:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985264256 00000100:00100000:2.0:1767319811.295555:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3168 00000100:00100000:2.0:1767319811.295563:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985264256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319811.295597:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985264256:12345-0@lo:400:kworker.0 Request processed in 35us (106us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319811.295603:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3168 00000100:00080000:2.0:1767319811.295628:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319811.295645:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985264256:0@lo:400:kworker.0 00000100:00100000:0.0:1767319811.295833:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985264000 00000100:00100000:0.0:1767319811.295838:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985264000 00000100:00100000:0.0:1767319811.295874:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4843 00000100:00100000:0.0:1767319811.295881:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985264000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319811.295915:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985264000:12345-0@lo:400:kworker.0 Request processed in 35us (480us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319811.295921:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4843 00000100:00080000:2.0:1767319811.298891:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319811.298953:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985264000:0@lo:400:kworker.0 00000100:00100000:3.0:1767319811.805650:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985264512:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319811.805721:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319811.805906:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985264512 00000100:00100000:2.0:1767319811.805914:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985264512 00000100:00100000:2.0:1767319811.805969:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2350 00000100:00100000:2.0:1767319811.805977:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985264512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319811.806003:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319811.806048:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985264512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (327us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319811.806055:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2350 00000100:00100000:3.0:1767319811.806156:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985264512:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319811.808087:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319811.808105:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985264640:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319811.808197:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319811.808954:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985264640 00000100:00100000:3.0:1767319811.808963:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985264640 00000100:00100000:3.0:1767319811.809017:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2351 00000100:00100000:3.0:1767319811.809026:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985264640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319811.809055:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319811.809123:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985264640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (917us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319811.809131:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2351 00000100:00100000:1.0:1767319811.809308:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985264640:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319811.931511:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319812.955440:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319813.980547:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319815.003387:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319815.343644:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767319815.343689:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920581760 offset 0 length 520: 4 00000100:00100000:0.0:1767319815.343736:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319815.343829:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582016 00000100:00100000:3.0:1767319815.343840:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582016 00000100:00100000:1.0:1767319815.343884:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920581888 00000100:00100000:1.0:1767319815.343890:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920581888 00000100:00100000:1.0:1767319815.343937:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4844 00000100:00100000:1.0:1767319815.343949:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920582016:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:2.0:1767319815.343955:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319815.344033:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920582016:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319815.344040:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4844 00000100:00100000:1.0:1767319815.344060:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5395 00000100:00100000:1.0:1767319815.344064:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920581888:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319815.344086:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319815.344112:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920581888:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (377us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319815.344118:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5395 00000100:00100000:1.0:1767319815.387305:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319790/real 1767319790] req@ffff8ce1a612d180 x1853163985259264/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319815 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319815.387339:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a612d180 x1853163985259264/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319815 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319815.387372:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319815.387403:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319815.387407:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985259264:0@lo:250:kworker.0 00000100:00080000:0.0:1767319815.387507:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.387513:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319815.387516:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.387521:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5039 00000100:00080000:0.0:1767319815.387524:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.387530:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319815.387561:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4000 x1853163985264768/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319815.387619:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985264768:0@lo:250:kworker.0 00000400:00000100:1.0:1767319815.387666:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985264768 offset 0 length 520: 4 00000100:00080000:0.0:1767319815.387724:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.387728:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.387730:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.387733:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.387735:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.387749:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319815.387767:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7100 x1853163985264896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319815.390854:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985264896:0@lo:38:kworker.0 00000100:00100000:1.0:1767319815.390928:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319815.391087:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985264896 00000100:00100000:3.0:1767319815.391096:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985264896 00000100:00100000:3.0:1767319815.391137:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5396 00000100:00100000:3.0:1767319815.391144:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985264896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319815.391185:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319815.391242:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985264896:12345-0@lo:38:kworker.0 Request processed in 96us (313us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319815.391249:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5396 00000100:00080000:2.0:1767319815.391377:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319815.391384:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319815.391387:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985264896:0@lo:38:kworker.0 00000100:00080000:0.0:1767319815.391471:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.391477:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.391479:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.391483:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.391487:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.391492:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319815.391520:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985265024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319815.391588:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767319815.391607:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.391611:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.391613:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.391616:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.391619:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.391622:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319815.391634:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4380 x1853163985265280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319815.391658:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.391661:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.391663:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.391666:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.391668:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.391671:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319815.391682:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca5180 x1853163985265536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319815.391732:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319815.391754:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985265792:0@lo:400:kworker.0 00000100:00100000:1.0:1767319815.391787:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319815.391816:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985265408:0@lo:400:kworker.0 00000100:00100000:1.0:1767319815.391838:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319815.391857:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319815.391862:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985265664:0@lo:400:kworker.0 00000100:00100000:1.0:1767319815.391883:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319815.391907:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265408 00000100:00100000:1.0:1767319815.391911:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265408 00000100:00100000:1.0:1767319815.391948:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4845 00000100:00100000:1.0:1767319815.391955:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985265408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319815.391995:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985265408:12345-0@lo:400:kworker.0 Request processed in 41us (155us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319815.392002:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4845 00000100:00100000:1.0:1767319815.392029:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985265408:0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.392116:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265792 00000100:00100000:2.0:1767319815.392120:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265792 00000100:00100000:2.0:1767319815.392156:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265664 00000100:00100000:2.0:1767319815.392159:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265664 00000100:00100000:2.0:1767319815.392199:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3169 00000100:00100000:2.0:1767319815.392205:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985265792:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.392239:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985265792:12345-0@lo:400:kworker.0 Request processed in 34us (452us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319815.392245:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3169 00000100:00100000:2.0:1767319815.392258:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3170 00000100:00100000:2.0:1767319815.392263:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985265664:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.392291:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985265664:12345-0@lo:400:kworker.0 Request processed in 28us (407us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319815.392297:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3170 00000100:00080000:2.0:1767319815.392320:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319815.392323:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319815.392326:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319815.392343:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985265792:0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.392364:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985265664:0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.392380:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985265024:0@lo:38:kworker.0 00000100:00100000:2.0:1767319815.392405:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319815.392424:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985265280:0@lo:38:kworker.0 00000100:00100000:2.0:1767319815.392443:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319815.392457:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985265536:0@lo:38:kworker.0 00000100:00100000:2.0:1767319815.392475:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319815.392503:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265024 00000100:00100000:2.0:1767319815.392506:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265024 00000100:00100000:2.0:1767319815.392527:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1839 00000100:00100000:2.0:1767319815.392530:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985265024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319815.392544:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319815.392573:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985265024:12345-0@lo:38:kworker.0 Request processed in 42us (168us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319815.392579:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1839 00000100:00100000:2.0:1767319815.392598:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265280 00000100:00100000:2.0:1767319815.392602:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265280 00000100:00100000:2.0:1767319815.392618:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265536 00000100:00100000:2.0:1767319815.392621:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265536 00000100:00100000:2.0:1767319815.392635:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5397 00000100:00100000:2.0:1767319815.392639:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985265280:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319815.392649:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319815.392672:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985265280:12345-0@lo:38:kworker.0 Request processed in 31us (228us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319815.392677:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5397 00000100:00100000:2.0:1767319815.392688:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5398 00000100:00100000:2.0:1767319815.392692:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985265536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319815.392700:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319815.392719:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985265536:12345-0@lo:38:kworker.0 Request processed in 26us (243us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319815.392725:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5398 00000100:00080000:2.0:1767319815.392761:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319815.392765:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319815.392768:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985265024:0@lo:38:kworker.0 00000100:00080000:2.0:1767319815.392775:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319815.392778:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319815.392780:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985265280:0@lo:38:kworker.0 00000100:00080000:2.0:1767319815.392787:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319815.392789:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319815.392792:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985265536:0@lo:38:kworker.0 00000100:00100000:3.0:1767319815.395864:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985265152:0@lo:400:kworker.0 00000100:00100000:3.0:1767319815.395944:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319815.396132:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265152 00000100:00100000:2.0:1767319815.396140:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265152 00000100:00100000:2.0:1767319815.396235:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4846 00000100:00100000:2.0:1767319815.396244:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985265152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.396300:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985265152:12345-0@lo:400:kworker.0 Request processed in 57us (355us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319815.396307:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4846 00000100:00080000:3.0:1767319815.396408:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319815.396438:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985265152:0@lo:400:kworker.0 00010000:00010000:1.0:1767319816.027370:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319816.923498:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985265920:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319816.923570:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319816.923684:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985265920 00000100:00100000:0.0:1767319816.923692:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985265920 00000100:00100000:0.0:1767319816.923760:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2352 00000100:00100000:0.0:1767319816.923769:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985265920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319816.923800:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319816.923861:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985265920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319816.923869:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2352 00000100:00100000:1.0:1767319816.923967:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985265920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319816.924563:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985266048:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319816.924601:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319816.924852:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266048 00000100:00100000:3.0:1767319816.924858:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266048 00000100:00100000:3.0:1767319816.924907:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2353 00000100:00100000:3.0:1767319816.924916:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985266048:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319816.924940:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319816.925287:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985266048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 370us (683us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319816.925320:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2353 00000100:00100000:1.0:1767319816.925418:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985266048:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319817.051771:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319818.078263:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319819.102118:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319820.123492:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319820.463455:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319820.463525:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319820.463669:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582272 00000100:00100000:2.0:1767319820.463683:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582272 00000100:00100000:1.0:1767319820.463697:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582144 00000100:00100000:1.0:1767319820.463702:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582144 00000100:00100000:1.0:1767319820.463765:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4847 00000100:00100000:1.0:1767319820.463779:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920582272:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319820.463817:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319820.463900:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920582272:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (460us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319820.463909:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4847 00000100:00100000:1.0:1767319820.463932:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5399 00000100:00100000:1.0:1767319820.463938:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920582144:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319820.463965:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319820.463993:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920582144:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (469us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319820.464000:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5399 00000100:00080000:2.0:1767319820.511063:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319820.511077:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319820.511080:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319820.511084:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319820.511090:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319820.511101:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5063 00000100:00080000:2.0:1767319820.511107:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319820.511118:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319820.511210:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d500 x1853163985266176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319820.511252:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319820.511256:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319820.511258:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319820.511262:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5063 00000100:00080000:2.0:1767319820.511265:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319820.511269:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319820.511285:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c380 x1853163985266304/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319820.511435:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985266432:0@lo:400:kworker.0 00000100:00100000:1.0:1767319820.511529:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319820.511654:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319820.511660:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319820.511662:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319820.511666:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5063 00000100:00080000:2.0:1767319820.511669:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319820.511674:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319820.511692:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985266560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319820.511734:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319820.511738:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319820.511740:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319820.511744:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5063 00000100:00080000:2.0:1767319820.511746:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319820.511750:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319820.511762:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612fb80 x1853163985266816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319820.511815:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985266944:0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.511859:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319820.511883:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319820.511889:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985266688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.511909:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319820.511932:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319820.511937:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985267072:0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.511959:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319820.511993:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266944 00000100:00100000:2.0:1767319820.512001:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266944 00000100:00100000:2.0:1767319820.512067:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267072 00000100:00100000:2.0:1767319820.512070:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267072 00000100:00100000:2.0:1767319820.512097:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3171 00000100:00100000:2.0:1767319820.512107:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985266944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.512209:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985266944:12345-0@lo:400:kworker.0 Request processed in 102us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319820.512218:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3171 00000100:00100000:2.0:1767319820.512240:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3172 00000100:00100000:2.0:1767319820.512246:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985267072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.512279:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985267072:12345-0@lo:400:kworker.0 Request processed in 33us (319us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319820.512285:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3172 00000100:00100000:2.0:1767319820.512336:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985266944:0@lo:400:kworker.0 00000100:00100000:1.0:1767319820.512581:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985267072:0@lo:400:kworker.0 00000100:00100000:0.0:1767319820.512722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985266176:0@lo:38:kworker.0 00000100:00100000:0.0:1767319820.512766:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319820.512801:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985266304:0@lo:38:kworker.0 00000100:00100000:0.0:1767319820.512832:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319820.512860:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985266560:0@lo:38:kworker.0 00000100:00100000:0.0:1767319820.512885:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319820.512904:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985266816:0@lo:38:kworker.0 00000100:00100000:0.0:1767319820.512925:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319820.512965:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266304 00000100:00100000:0.0:1767319820.512971:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266304 00000100:00100000:0.0:1767319820.513003:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1840 00000100:00100000:0.0:1767319820.513010:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985266304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319820.513038:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319820.513116:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985266304:12345-0@lo:38:kworker.0 Request processed in 98us (276us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319820.513123:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1840 00000100:00100000:0.0:1767319820.513148:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266432 00000100:00100000:0.0:1767319820.513152:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266432 00000100:00100000:1.0:1767319820.513461:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266688 00000100:00100000:1.0:1767319820.513466:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266688 00000100:00100000:1.0:1767319820.513501:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266176 00000100:00100000:1.0:1767319820.513505:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266176 00000100:00100000:1.0:1767319820.513515:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266560 00000100:00100000:1.0:1767319820.513517:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266560 00000100:00100000:1.0:1767319820.513525:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985266816 00000100:00100000:1.0:1767319820.513528:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985266816 00000100:00100000:1.0:1767319820.513538:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4848 00000100:00100000:1.0:1767319820.513544:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985266688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319820.513594:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985266688:12345-0@lo:400:kworker.0 Request processed in 49us (1683us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319820.513600:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4848 00000100:00100000:1.0:1767319820.513616:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5400 00000100:00100000:1.0:1767319820.513620:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985266176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319820.513632:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319820.513670:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985266176:12345-0@lo:38:kworker.0 Request processed in 48us (903us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319820.513676:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5400 00000100:00100000:1.0:1767319820.513688:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5401 00000100:00100000:1.0:1767319820.513692:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985266560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319820.513701:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319820.513723:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985266560:12345-0@lo:38:kworker.0 Request processed in 29us (837us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319820.513729:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5401 00000100:00100000:1.0:1767319820.513739:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5402 00000100:00100000:1.0:1767319820.513743:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985266816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319820.513752:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319820.513775:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985266816:12345-0@lo:38:kworker.0 Request processed in 30us (849us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319820.513782:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5402 00000100:00080000:1.0:1767319820.513967:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319820.513973:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319820.513977:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985266176:0@lo:38:kworker.0 00000100:00080000:1.0:1767319820.513989:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319820.513992:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319820.513994:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985266304:0@lo:38:kworker.0 00000100:00080000:1.0:1767319820.514003:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319820.514006:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319820.514009:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985266560:0@lo:38:kworker.0 00000100:00080000:1.0:1767319820.514017:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319820.514020:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319820.514023:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985266816:0@lo:38:kworker.0 00000100:00100000:0.0:1767319820.514306:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4849 00000100:00100000:0.0:1767319820.514314:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985266432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.514489:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985266688:0@lo:400:kworker.0 00000100:00080000:3.0:1767319820.514601:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319820.514608:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319820.514611:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319820.515657:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985266432:12345-0@lo:400:kworker.0 Request processed in 1325us (4111us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319820.515671:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4849 00000100:00100000:2.0:1767319820.516145:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985266432:0@lo:400:kworker.0 00010000:00010000:2.0:1767319821.147461:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319822.043960:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319822.043987:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985267328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319822.043987:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985267200:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319822.048856:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319822.049184:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267200 00000100:00100000:0.0:1767319822.049197:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267200 00000100:00100000:0.0:1767319822.049483:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2354 00000100:00100000:0.0:1767319822.049495:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985267200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319822.049540:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319822.049632:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985267200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 139us (788us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319822.049641:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2354 00000100:00100000:0.0:1767319822.049722:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985267200:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319822.050559:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319822.055980:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319822.060852:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267328 00000100:00100000:1.0:1767319822.060877:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267328 00000100:00080000:3.0:1767319822.064634:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319822.064779:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2355 00000100:00100000:0.0:1767319822.064799:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985267328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319822.064849:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319822.064976:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985267328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 185us (9008us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319822.064985:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2355 00000100:00100000:2.0:1767319822.065679:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985267328:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319822.173397:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319823.197748:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319824.219475:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319825.246418:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319825.588877:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319825.588983:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319825.589179:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582528 00000100:00100000:0.0:1767319825.589191:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582528 00000100:00100000:0.0:1767319825.589274:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582400 00000100:00100000:0.0:1767319825.589278:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582400 00000100:00100000:0.0:1767319825.589297:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4850 00000100:00100000:0.0:1767319825.589309:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920582528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319825.589385:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920582528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (518us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319825.589394:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4850 00000100:00100000:0.0:1767319825.589413:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5403 00000100:00100000:0.0:1767319825.589419:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920582400:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319825.589440:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319825.589467:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920582400:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (485us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319825.589475:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5403 00000100:00080000:3.0:1767319825.589598:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319825.627463:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319825.627474:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319825.627477:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319825.627481:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319825.627486:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319825.627495:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5068 00000100:00080000:0.0:1767319825.627499:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319825.627506:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319825.627548:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985267456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319825.627575:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319825.627580:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319825.627582:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319825.627585:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5068 00000100:00080000:0.0:1767319825.627588:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319825.627591:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319825.627604:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07480 x1853163985267584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319825.627658:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985267456:0@lo:38:kworker.0 00000100:00100000:2.0:1767319825.627724:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319825.627864:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267456 00000100:00100000:3.0:1767319825.627874:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267456 00000100:00100000:3.0:1767319825.627915:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5404 00000100:00100000:3.0:1767319825.627922:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985267456:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319825.627941:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319825.627982:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985267456:12345-0@lo:38:kworker.0 Request processed in 59us (257us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319825.627993:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5404 00000100:00080000:1.0:1767319825.628139:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319825.628146:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319825.628150:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985267456:0@lo:38:kworker.0 00000100:00100000:3.0:1767319825.628345:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767319825.628365:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319825.628368:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319825.628370:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319825.628374:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5068 00000100:00080000:0.0:1767319825.628377:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319825.628380:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319825.628395:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985267840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319825.628421:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319825.628424:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319825.628426:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319825.628429:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5068 00000100:00080000:0.0:1767319825.628431:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319825.628434:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319825.628444:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73df80 x1853163985268096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319825.628501:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985267968:0@lo:400:kworker.0 00000100:00100000:1.0:1767319825.628532:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319825.628568:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319825.628574:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985268224:0@lo:400:kworker.0 00000100:00100000:1.0:1767319825.628600:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319825.628627:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268224 00000100:00100000:1.0:1767319825.628631:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268224 00000100:00100000:2.0:1767319825.628798:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319825.628805:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985268352:0@lo:400:kworker.0 00000100:00100000:2.0:1767319825.628824:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319825.628845:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268352 00000100:00100000:2.0:1767319825.628848:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268352 00000100:00100000:2.0:1767319825.628889:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3173 00000100:00100000:2.0:1767319825.628897:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985268352:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319825.628937:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985268352:12345-0@lo:400:kworker.0 Request processed in 41us (111us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319825.628943:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3173 00000100:00080000:2.0:1767319825.628965:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319825.628984:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985268352:0@lo:400:kworker.0 00000100:00100000:2.0:1767319825.629008:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267968 00000100:00100000:2.0:1767319825.629010:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267968 00000100:00100000:2.0:1767319825.629031:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4851 00000100:00100000:2.0:1767319825.629045:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985267968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319825.629082:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985267968:12345-0@lo:400:kworker.0 Request processed in 47us (549us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319825.629089:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4851 00000100:00100000:2.0:1767319825.629114:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985267968:0@lo:400:kworker.0 00000100:00080000:2.0:1767319825.629129:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319825.629215:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3174 00000100:00100000:1.0:1767319825.629220:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985268224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319825.629255:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985268224:12345-0@lo:400:kworker.0 Request processed in 34us (654us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319825.629260:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3174 00000100:00100000:1.0:1767319825.629285:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985268224:0@lo:400:kworker.0 00000100:00100000:1.0:1767319825.629301:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985267584:0@lo:38:kworker.0 00000100:00100000:1.0:1767319825.629327:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319825.629350:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985267840:0@lo:38:kworker.0 00000100:00100000:1.0:1767319825.629370:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319825.629384:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985268096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319825.629402:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319825.629429:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267584 00000100:00100000:1.0:1767319825.629433:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267584 00000100:00100000:1.0:1767319825.629452:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1841 00000100:00100000:1.0:1767319825.629456:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985267584:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319825.629470:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319825.629500:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985267584:12345-0@lo:38:kworker.0 Request processed in 42us (173us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319825.629506:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1841 00000100:00100000:1.0:1767319825.629524:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267840 00000100:00100000:1.0:1767319825.629527:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267840 00000100:00100000:1.0:1767319825.629541:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268096 00000100:00100000:1.0:1767319825.629544:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268096 00000100:00100000:1.0:1767319825.629570:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5405 00000100:00100000:1.0:1767319825.629574:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985267840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319825.629584:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319825.629607:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985267840:12345-0@lo:38:kworker.0 Request processed in 31us (236us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319825.629613:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5405 00000100:00100000:1.0:1767319825.629620:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5406 00000100:00100000:1.0:1767319825.629623:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985268096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319825.629628:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319825.629643:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985268096:12345-0@lo:38:kworker.0 Request processed in 20us (241us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319825.629646:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5406 00000100:00080000:1.0:1767319825.629663:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319825.629665:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319825.629667:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985267584:0@lo:38:kworker.0 00000100:00080000:1.0:1767319825.629672:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319825.629674:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319825.629675:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985267840:0@lo:38:kworker.0 00000100:00080000:1.0:1767319825.629679:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319825.629681:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319825.629682:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985268096:0@lo:38:kworker.0 00000100:00080000:2.0:1767319825.629739:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319825.631177:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985267712:0@lo:400:kworker.0 00000100:00100000:3.0:1767319825.631229:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319825.631340:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985267712 00000100:00100000:1.0:1767319825.631349:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985267712 00000100:00100000:1.0:1767319825.631418:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4852 00000100:00100000:1.0:1767319825.631426:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985267712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319825.631473:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985267712:12345-0@lo:400:kworker.0 Request processed in 48us (244us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319825.631480:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4852 00000100:00080000:2.0:1767319825.631544:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319825.632141:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985267712:0@lo:400:kworker.0 00010000:00010000:1.0:1767319826.267484:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319827.163475:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985268480:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319827.163568:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319827.163684:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268480 00000100:00100000:2.0:1767319827.163692:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268480 00000100:00100000:2.0:1767319827.163764:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2356 00000100:00100000:2.0:1767319827.163775:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985268480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319827.163806:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319827.163862:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985268480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (293us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319827.163869:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2356 00000100:00100000:2.0:1767319827.164262:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985268608:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319827.164292:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319827.164481:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319827.164495:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268608 00000100:00100000:0.0:1767319827.164499:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268608 00000100:00100000:0.0:1767319827.164554:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2357 00000100:00100000:0.0:1767319827.164562:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985268608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319827.164579:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319827.164610:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985268608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (317us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319827.164616:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2357 00000100:00100000:0.0:1767319827.164658:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985268480:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319827.164999:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319827.165023:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985268608:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319827.294861:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319828.315760:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319829.340050:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319830.363550:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319830.700322:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319830.700384:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319830.700773:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582784 00000100:00100000:0.0:1767319830.700827:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582784 00000100:00100000:0.0:1767319830.700898:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582656 00000100:00100000:0.0:1767319830.700904:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582656 00000100:00080000:3.0:1767319830.700928:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319830.700938:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4853 00000100:00100000:0.0:1767319830.700952:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920582784:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319830.700996:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5407 00000100:00100000:3.0:1767319830.701008:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920582656:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319830.701039:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319830.701042:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920582784:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (734us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319830.701052:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4853 00000100:00100000:3.0:1767319830.701102:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920582656:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (714us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319830.701112:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5407 00000100:00080000:1.0:1767319830.748847:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319830.748868:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319830.748871:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319830.748875:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319830.748879:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319830.748887:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5073 00000100:00080000:1.0:1767319830.748892:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319830.748910:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319830.748950:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518000 x1853163985268736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319830.748976:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319830.748979:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319830.748981:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319830.748984:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5073 00000100:00080000:1.0:1767319830.748986:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319830.748990:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319830.749003:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519180 x1853163985268864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319830.749053:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319830.749057:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319830.749059:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319830.749062:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5073 00000100:00080000:1.0:1767319830.749065:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319830.749068:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319830.749082:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05f80 x1853163985269120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319830.749111:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319830.749115:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319830.749117:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319830.749120:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5073 00000100:00080000:1.0:1767319830.749122:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319830.749125:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319830.749135:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07480 x1853163985269376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319830.749199:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985268992:0@lo:400:kworker.0 00000100:00100000:1.0:1767319830.749201:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985269248:0@lo:400:kworker.0 00000100:00100000:0.0:1767319830.749300:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985269632:0@lo:400:kworker.0 00000100:00100000:2.0:1767319830.749316:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985268736:0@lo:38:kworker.0 00000100:00100000:3.0:1767319830.751206:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319830.751361:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268992 00000100:00100000:3.0:1767319830.751370:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268992 00000100:00100000:3.0:1767319830.751428:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4854 00000100:00100000:3.0:1767319830.751438:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985268992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319830.751511:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985268992:12345-0@lo:400:kworker.0 Request processed in 74us (312us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319830.751519:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4854 00000100:00080000:3.0:1767319830.751873:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319830.751921:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985268992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319830.752091:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985269504:0@lo:400:kworker.0 00000100:00100000:3.0:1767319830.752133:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319830.752211:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985269504 00000100:00100000:3.0:1767319830.752217:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985269504 00000100:00100000:3.0:1767319830.752268:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3175 00000100:00100000:3.0:1767319830.752275:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985269504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319830.752311:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985269504:12345-0@lo:400:kworker.0 Request processed in 36us (178us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319830.752318:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3175 00000100:00080000:3.0:1767319830.752334:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319830.752351:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985269504:0@lo:400:kworker.0 00000100:00100000:0.0:1767319830.756132:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319830.756403:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985269632 00000100:00100000:0.0:1767319830.756411:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985269632 00000100:00100000:1.0:1767319830.756571:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319830.756615:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985269248 00000100:00100000:1.0:1767319830.756619:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985269248 00000100:00100000:1.0:1767319830.756654:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4855 00000100:00100000:1.0:1767319830.756664:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985269248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319830.756720:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985269248:12345-0@lo:400:kworker.0 Request processed in 58us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319830.756728:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4855 00000100:00100000:1.0:1767319830.756771:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985269248:0@lo:400:kworker.0 00000100:00080000:1.0:1767319830.756808:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319830.756812:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319830.758725:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319830.758782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985268864:0@lo:38:kworker.0 00000100:00100000:2.0:1767319830.758821:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319830.759703:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985269120:0@lo:38:kworker.0 00000100:00100000:2.0:1767319830.759740:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319830.759764:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985269376:0@lo:38:kworker.0 00000100:00100000:2.0:1767319830.759784:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319830.759826:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268736 00000100:00100000:2.0:1767319830.759833:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268736 00000100:00100000:2.0:1767319830.759907:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985269120 00000100:00100000:2.0:1767319830.759913:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985269120 00000100:00100000:2.0:1767319830.759934:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985269376 00000100:00100000:2.0:1767319830.759937:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985269376 00000100:00100000:2.0:1767319830.759960:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5408 00000100:00100000:2.0:1767319830.759968:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985268736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319830.759996:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319830.760040:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985268736:12345-0@lo:38:kworker.0 Request processed in 72us (1322us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319830.760046:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5408 00000100:00100000:2.0:1767319830.760063:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5409 00000100:00100000:2.0:1767319830.760066:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985269120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319830.760077:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319830.760099:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985269120:12345-0@lo:38:kworker.0 Request processed in 31us (358us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319830.760104:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5409 00000100:00100000:2.0:1767319830.760114:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5410 00000100:00100000:2.0:1767319830.760118:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985269376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319830.760127:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319830.760147:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985269376:12345-0@lo:38:kworker.0 Request processed in 27us (361us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319830.760152:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5410 00000100:00080000:2.0:1767319830.760209:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319830.760214:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319830.760217:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985268736:0@lo:38:kworker.0 00000100:00080000:2.0:1767319830.760226:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319830.760230:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319830.760232:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985269120:0@lo:38:kworker.0 00000100:00080000:2.0:1767319830.760238:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319830.760240:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319830.760243:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985269376:0@lo:38:kworker.0 00000100:00100000:0.0:1767319830.761401:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3176 00000100:00100000:0.0:1767319830.761419:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985269632:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319830.761500:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985269632:12345-0@lo:400:kworker.0 Request processed in 89us (5376us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319830.761508:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3176 00000100:00100000:0.0:1767319830.761557:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985269632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319830.765599:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985268864 00000100:00100000:3.0:1767319830.765616:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985268864 00000100:00100000:3.0:1767319830.765685:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1842 00000100:00100000:3.0:1767319830.765694:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985268864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319830.765720:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319830.765817:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985268864:12345-0@lo:38:kworker.0 Request processed in 123us (6995us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319830.765825:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1842 00000100:00080000:2.0:1767319830.766212:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319830.766220:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319830.766224:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985268864:0@lo:38:kworker.0 00010000:00010000:1.0:1767319831.392954:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319832.287051:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319832.287078:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985269760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319832.287149:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319832.287271:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985269760 00000100:00100000:2.0:1767319832.287281:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985269760 00000100:00100000:2.0:1767319832.287366:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2358 00000100:00100000:2.0:1767319832.287377:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985269760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319832.287405:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319832.287461:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985269760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319832.287467:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2358 00000100:00100000:2.0:1767319832.287512:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985269760:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319832.288386:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319832.289108:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319832.289129:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985269888:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319832.289212:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319832.290129:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985269888 00000100:00100000:1.0:1767319832.290139:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985269888 00000100:00100000:1.0:1767319832.290230:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2359 00000100:00100000:1.0:1767319832.290241:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985269888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319832.290267:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319832.290325:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985269888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (1112us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319832.290332:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2359 00000100:00080000:0.0:1767319832.291187:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319832.291764:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985269888:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319832.411645:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319833.435729:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319834.460197:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319835.483617:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319835.822644:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319835.823275:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319835.823472:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920583040 00000100:00100000:0.0:1767319835.823483:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920583040 00000100:00100000:0.0:1767319835.823569:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920582912 00000100:00100000:0.0:1767319835.823573:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920582912 00000100:00100000:0.0:1767319835.823593:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4856 00000100:00100000:0.0:1767319835.823603:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920583040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319835.823673:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920583040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 72us (1037us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319835.823681:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4856 00000100:00100000:0.0:1767319835.823700:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5411 00000100:00100000:0.0:1767319835.823704:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920582912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319835.823723:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319835.823745:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920582912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 39us (473us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319835.823751:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5411 00000100:00080000:2.0:1767319835.825157:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319835.867267:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.867277:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319835.867280:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.867283:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.867288:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.867294:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.867299:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.867306:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319835.867342:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985270016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319835.867371:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.867375:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.867377:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.867380:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.867382:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.867385:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319835.867399:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c380 x1853163985270144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319835.867400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985270016:0@lo:38:kworker.0 00000100:00080000:1.0:1767319835.867439:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.867442:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.867444:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.867447:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.867449:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.867452:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319835.867461:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319835.867461:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d180 x1853163985270400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319835.867485:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.867489:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.867490:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.867493:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.867495:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.867498:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319835.867509:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612e680 x1853163985270656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319835.867547:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985270272:0@lo:400:kworker.0 00000100:00100000:3.0:1767319835.867573:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.867621:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985270144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319835.867647:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319835.874021:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985270528:0@lo:400:kworker.0 00000100:00100000:1.0:1767319835.874092:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.875086:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985270400:0@lo:38:kworker.0 00000100:00100000:2.0:1767319835.875095:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270016 00000100:00100000:2.0:1767319835.875104:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270016 00000100:00100000:0.0:1767319835.875132:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.875153:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985270656:0@lo:38:kworker.0 00000100:00100000:2.0:1767319835.875160:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270272 00000100:00100000:2.0:1767319835.875176:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270272 00000100:00100000:0.0:1767319835.875188:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319835.875207:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270528 00000100:00100000:2.0:1767319835.875210:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270528 00000100:00100000:2.0:1767319835.875222:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270400 00000100:00100000:2.0:1767319835.875224:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270400 00000100:00100000:0.0:1767319835.875228:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270144 00000100:00100000:0.0:1767319835.875231:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270144 00000100:00100000:2.0:1767319835.875232:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270656 00000100:00100000:2.0:1767319835.875234:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270656 00000100:00100000:2.0:1767319835.875243:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4857 00000100:00100000:0.0:1767319835.875251:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1843 00000100:00100000:2.0:1767319835.875251:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985270272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319835.875256:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985270144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319835.875274:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319835.875299:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985270272:12345-0@lo:400:kworker.0 Request processed in 49us (7725us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319835.875303:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985270144:12345-0@lo:38:kworker.0 Request processed in 47us (7656us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319835.875306:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4857 00000100:00100000:0.0:1767319835.875309:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1843 00000100:00100000:2.0:1767319835.875322:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4858 00000100:00100000:2.0:1767319835.875327:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985270528:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319835.875346:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319835.875351:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.875355:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985270144:0@lo:38:kworker.0 00000100:00100000:2.0:1767319835.875355:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985270528:12345-0@lo:400:kworker.0 Request processed in 28us (1263us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319835.875374:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4858 00000100:00100000:2.0:1767319835.875385:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5412 00000100:00100000:2.0:1767319835.875388:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985270016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319835.875398:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319835.875408:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5413 00000100:00100000:0.0:1767319835.875412:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985270400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319835.875421:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319835.875423:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985270016:12345-0@lo:38:kworker.0 Request processed in 34us (7963us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319835.875429:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5412 00000100:00100000:2.0:1767319835.875439:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5414 00000100:00100000:0.0:1767319835.875441:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985270400:12345-0@lo:38:kworker.0 Request processed in 28us (309us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319835.875443:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985270656:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319835.875447:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5413 00010000:00020000:2.0:1767319835.875451:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319835.875466:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767319835.875469:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985270656:12345-0@lo:38:kworker.0 Request processed in 25us (280us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767319835.875470:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.875473:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985270400:0@lo:38:kworker.0 00000100:00100000:2.0:1767319835.875475:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5414 00000100:00080000:0.0:1767319835.875480:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319835.875483:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.875485:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985270656:0@lo:38:kworker.0 00000100:00080000:0.0:1767319835.875493:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319835.875496:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.875498:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985270016:0@lo:38:kworker.0 00000100:00080000:2.0:1767319835.875506:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319835.875518:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985270912:0@lo:400:kworker.0 00000100:00100000:0.0:1767319835.875535:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985270784:0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.875542:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.875554:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319835.875570:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270912 00000100:00100000:2.0:1767319835.875573:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270912 00000100:00100000:0.0:1767319835.875575:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985270784 00000100:00100000:0.0:1767319835.875577:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985270784 00000100:00100000:2.0:1767319835.875601:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3177 00000100:00100000:2.0:1767319835.875606:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985270912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.875638:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985270912:12345-0@lo:400:kworker.0 Request processed in 32us (95us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319835.875643:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3177 00000100:00100000:3.0:1767319835.877780:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985270272:0@lo:400:kworker.0 00000100:00080000:3.0:1767319835.877843:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319835.877848:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319835.878406:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985270912:0@lo:400:kworker.0 00000100:00100000:1.0:1767319835.878528:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985270528:0@lo:400:kworker.0 00000100:00100000:0.0:1767319835.879409:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3178 00000100:00100000:0.0:1767319835.879428:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985270784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319835.879570:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985270784:12345-0@lo:400:kworker.0 Request processed in 152us (4014us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319835.879578:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3178 00000100:00100000:1.0:1767319835.879678:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985270784:0@lo:400:kworker.0 00010000:00010000:1.0:1767319836.508789:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319837.403377:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319837.403401:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985271168:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319837.403478:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319837.403524:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985271168 00000100:00100000:2.0:1767319837.403533:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985271168 00000100:00100000:3.0:1767319837.403570:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319837.403580:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985271040:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319837.403601:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2360 00000100:00100000:3.0:1767319837.403607:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319837.403610:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985271168:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319837.403643:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985271040 00000100:00100000:3.0:1767319837.403647:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985271040 00000100:00100000:3.0:1767319837.403680:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2361 00000100:00100000:3.0:1767319837.403687:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985271040:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319837.403703:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319837.403706:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:3.0:1767319837.403714:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319837.403753:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985271040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (144us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319837.403760:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2361 00000100:00100000:1.0:1767319837.403781:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985271040:0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319837.404227:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319837.404276:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985271168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 668us (799us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319837.404283:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2360 00000100:00100000:1.0:1767319837.404396:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985271168:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319837.532669:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319838.555437:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319839.583649:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319840.606467:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319840.939971:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319840.940010:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920583168 offset 0 length 520: 4 00000100:00100000:2.0:1767319840.940039:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319840.940843:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920583424 00000100:00100000:1.0:1767319840.940862:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920583424 00000100:00100000:3.0:1767319840.940969:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920583296 00000100:00100000:3.0:1767319840.940982:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920583296 00000100:00100000:3.0:1767319840.941045:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4859 00000100:00100000:3.0:1767319840.941061:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920583424:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319840.941138:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920583424:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (1177us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319840.941147:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4859 00000100:00100000:3.0:1767319840.941202:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5415 00000100:00100000:3.0:1767319840.941207:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920583296:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319840.941227:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319840.941251:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920583296:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (1212us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319840.941277:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5415 00000100:00080000:2.0:1767319840.946049:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319840.987286:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319815/real 1767319815] req@ffff8ce19dca4000 x1853163985264768/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319840.987319:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19dca4000 x1853163985264768/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319840.987340:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319840.987372:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319840.987377:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985264768:0@lo:250:kworker.0 00000100:00080000:2.0:1767319840.987785:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319840.987792:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319840.987797:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319840.987811:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5063 00000100:00080000:2.0:1767319840.987816:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319840.987823:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319840.987862:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1f80 x1853163985271296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319840.987891:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319840.987894:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319840.987896:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319840.987899:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5084 00000100:00080000:2.0:1767319840.987901:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319840.987904:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319840.987919:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0000 x1853163985271424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319840.987928:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319840.987932:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319840.987934:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319840.987937:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5084 00000100:00080000:2.0:1767319840.987939:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319840.987943:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319840.987955:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0380 x1853163985271552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319840.988000:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319840.988004:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319840.988006:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319840.988009:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5084 00000100:00080000:2.0:1767319840.988012:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319840.988015:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319840.988025:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0700 x1853163985271808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319840.988045:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319840.988049:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319840.988050:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319840.988053:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5084 00000100:00080000:2.0:1767319840.988056:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319840.988063:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319840.988075:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0e00 x1853163985272064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319840.988201:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985271680:0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.988258:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319840.988327:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319840.988334:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985272320:0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.988358:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319840.988383:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985271936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.988404:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319840.988419:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319840.988423:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985272192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.988442:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319840.988473:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272320 00000100:00100000:1.0:1767319840.988480:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272320 00000100:00100000:1.0:1767319840.988526:0:18371:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272192 00000100:00100000:1.0:1767319840.988529:0:18371:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272192 00000100:00100000:1.0:1767319840.988552:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3179 00000100:00100000:1.0:1767319840.988561:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985272320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.988619:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985272320:12345-0@lo:400:kworker.0 Request processed in 59us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319840.988625:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3179 00000100:00100000:1.0:1767319840.988640:0:18371:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3180 00000100:00100000:1.0:1767319840.988645:0:18371:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985272192:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.988671:0:18371:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985272192:12345-0@lo:400:kworker.0 Request processed in 26us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319840.988676:0:18371:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3180 00000100:00080000:1.0:1767319840.988689:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319840.988693:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319840.988717:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985272192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.988736:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985272320:0@lo:400:kworker.0 00000100:00100000:0.0:1767319840.988939:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985271296:0@lo:250:kworker.0 00000400:00000100:0.0:1767319840.988962:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985271296 offset 0 length 520: 4 00000100:00100000:0.0:1767319840.988981:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985271424:0@lo:38:kworker.0 00000100:00100000:0.0:1767319840.989000:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319840.989028:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985271552:0@lo:38:kworker.0 00000100:00100000:0.0:1767319840.989047:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319840.989064:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985271808:0@lo:38:kworker.0 00000100:00100000:0.0:1767319840.989084:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319840.989098:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985272064:0@lo:38:kworker.0 00000100:00100000:0.0:1767319840.989116:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319840.989154:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985271552 00000100:00100000:0.0:1767319840.989158:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985271552 00000100:00100000:0.0:1767319840.989204:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1844 00000100:00100000:0.0:1767319840.989209:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985271552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319840.989228:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319840.989261:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985271552:12345-0@lo:38:kworker.0 Request processed in 50us (213us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319840.989276:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1844 00000100:00100000:0.0:1767319840.989296:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985271680 00000100:00100000:0.0:1767319840.989299:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985271680 00000100:00100000:0.0:1767319840.989325:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985271936 00000100:00100000:0.0:1767319840.989328:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985271936 00000100:00100000:0.0:1767319840.989340:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985271424 00000100:00100000:0.0:1767319840.989343:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985271424 00000100:00100000:0.0:1767319840.989351:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985271808 00000100:00100000:0.0:1767319840.989355:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985271808 00000100:00100000:0.0:1767319840.989361:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272064 00000100:00100000:0.0:1767319840.989364:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272064 00000100:00100000:0.0:1767319840.989372:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4860 00000100:00100000:0.0:1767319840.989378:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985271680:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319840.989413:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985271680:12345-0@lo:400:kworker.0 Request processed in 35us (1154us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319840.989419:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4860 00000100:00100000:0.0:1767319840.989429:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4861 00000100:00100000:0.0:1767319840.989435:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985271936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319840.989462:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985271936:12345-0@lo:400:kworker.0 Request processed in 27us (1058us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319840.989468:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4861 00000100:00100000:0.0:1767319840.989480:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5416 00000100:00100000:0.0:1767319840.989485:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985271424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319840.989494:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319840.989516:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985271424:12345-0@lo:38:kworker.0 Request processed in 31us (516us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319840.989522:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5416 00000100:00100000:0.0:1767319840.989532:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5417 00000100:00100000:0.0:1767319840.989536:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985271808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319840.989544:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319840.989563:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985271808:12345-0@lo:38:kworker.0 Request processed in 26us (479us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319840.989568:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5417 00000100:00100000:0.0:1767319840.989716:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5418 00000100:00100000:0.0:1767319840.989721:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985272064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319840.989731:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319840.989756:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985272064:12345-0@lo:38:kworker.0 Request processed in 33us (639us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319840.989761:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5418 00000100:00080000:0.0:1767319840.989798:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319840.989803:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319840.989806:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985271424:0@lo:38:kworker.0 00000100:00080000:0.0:1767319840.989814:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319840.989818:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319840.989821:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985271552:0@lo:38:kworker.0 00000100:00080000:0.0:1767319840.989828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319840.989831:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319840.989834:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985271808:0@lo:38:kworker.0 00000100:00080000:0.0:1767319840.989841:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319840.989844:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319840.989847:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985272064:0@lo:38:kworker.0 00000100:00080000:1.0:1767319840.990156:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319840.990209:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985271936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319840.990232:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985271680:0@lo:400:kworker.0 00010000:00010000:1.0:1767319841.627434:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319842.524001:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985272448:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319842.524115:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319842.524183:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319842.524198:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985272576:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319842.525258:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319842.531858:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272448 00000100:00100000:2.0:1767319842.531874:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272448 00000100:00100000:2.0:1767319842.531963:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272576 00000100:00100000:2.0:1767319842.531967:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272576 00000100:00100000:2.0:1767319842.531995:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2362 00000100:00100000:2.0:1767319842.532006:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985272448:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319842.532042:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319842.532125:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985272448:12345-0@lo:13:osp-pre-0-1.0 Request processed in 121us (8009us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319842.532133:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2362 00000100:00100000:2.0:1767319842.532151:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2363 00000100:00100000:2.0:1767319842.532158:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985272576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319842.532203:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319842.533828:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985272448:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319842.534704:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319842.534711:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319842.534877:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985272576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2716us (9618us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319842.534889:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2363 00000100:00100000:1.0:1767319842.535082:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985272576:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319842.651365:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319843.676908:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319844.700786:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319845.723413:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319846.060946:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319846.061037:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319846.061122:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920583552 00000100:00100000:2.0:1767319846.061133:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920583552 00000100:00100000:1.0:1767319846.061233:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920583680 00000100:00100000:1.0:1767319846.061239:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920583680 00000100:00100000:2.0:1767319846.061244:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5419 00000100:00100000:2.0:1767319846.061262:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920583552:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319846.061287:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319846.061321:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4862 00000100:00100000:1.0:1767319846.061331:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920583680:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319846.061364:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920583552:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 99us (425us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319846.061376:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5419 00000100:00100000:1.0:1767319846.061382:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920583680:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 51us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319846.061391:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4862 00000100:00080000:0.0:1767319846.061463:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319846.107618:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319846.107626:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319846.107629:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319846.107633:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319846.107636:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319846.107645:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5089 00000100:00080000:2.0:1767319846.107650:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319846.107656:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319846.107694:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1180 x1853163985272704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319846.108233:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985272704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319846.108311:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319846.108371:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272704 00000100:00100000:3.0:1767319846.108380:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272704 00000100:00100000:3.0:1767319846.108422:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5420 00000100:00100000:3.0:1767319846.108431:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985272704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319846.108452:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319846.108499:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319846.108504:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319846.108507:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319846.108511:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5089 00000100:00080000:2.0:1767319846.108513:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319846.108517:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319846.108538:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1880 x1853163985272832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319846.109674:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985272832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319846.109730:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319846.109808:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272832 00000100:00100000:1.0:1767319846.109816:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272832 00000100:00100000:1.0:1767319846.109850:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1845 00000100:00100000:1.0:1767319846.109857:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985272832:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319846.109877:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319846.110430:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319846.110435:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319846.110439:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319846.110444:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5089 00000100:00080000:2.0:1767319846.110447:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319846.110451:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319846.110468:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0a80 x1853163985273088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319846.110497:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319846.110501:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319846.110503:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319846.110506:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5089 00000100:00080000:2.0:1767319846.110508:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319846.110511:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319846.110521:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985273344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319846.110560:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985273600:0@lo:400:kworker.0 00000100:00100000:2.0:1767319846.110599:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319846.110628:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319846.110633:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985272960:0@lo:400:kworker.0 00000100:00100000:2.0:1767319846.110656:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319846.110685:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273600 00000100:00100000:2.0:1767319846.110689:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273600 00000100:00100000:2.0:1767319846.110731:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3181 00000100:00100000:2.0:1767319846.110738:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985273600:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319846.110786:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985273600:12345-0@lo:400:kworker.0 Request processed in 48us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319846.110794:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3181 00000100:00100000:1.0:1767319846.111882:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985272832:12345-0@lo:38:kworker.0 Request processed in 2019us (2146us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319846.112330:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1845 00000100:00100000:1.0:1767319846.112406:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985273472:0@lo:400:kworker.0 00000100:00100000:1.0:1767319846.112444:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319846.114487:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319846.114502:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319846.114507:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985272704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319846.114545:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985273088:0@lo:38:kworker.0 00000100:00100000:0.0:1767319846.114602:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319846.114635:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985273344:0@lo:38:kworker.0 00000100:00100000:3.0:1767319846.114643:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985272704:12345-0@lo:38:kworker.0 Request processed in 6206us (6326us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319846.114657:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5420 00000100:00100000:0.0:1767319846.114662:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319846.114691:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319846.114691:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985272960 00000100:00080000:0.0:1767319846.114698:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319846.114700:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985272960 00000100:00100000:0.0:1767319846.114703:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985272832:0@lo:38:kworker.0 00000100:00080000:0.0:1767319846.114733:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319846.114739:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319846.114756:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273088 00000100:00100000:3.0:1767319846.114760:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273088 00000100:00100000:3.0:1767319846.114789:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273344 00000100:00100000:3.0:1767319846.114793:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273344 00000100:00100000:0.0:1767319846.114807:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4863 00000100:00100000:3.0:1767319846.114815:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5421 00000100:00100000:0.0:1767319846.114820:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985272960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319846.114825:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985273088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319846.115022:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319846.115047:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985272960:12345-0@lo:400:kworker.0 Request processed in 230us (4388us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319846.115056:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4863 00000100:00100000:3.0:1767319846.115071:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985273088:12345-0@lo:38:kworker.0 Request processed in 247us (467us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319846.115079:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5421 00000100:00100000:0.0:1767319846.115085:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5422 00000100:00100000:0.0:1767319846.115091:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985273344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319846.115103:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319846.115109:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985273216:0@lo:400:kworker.0 00000100:00100000:0.0:1767319846.115136:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985273344:12345-0@lo:38:kworker.0 Request processed in 43us (473us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319846.115146:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5422 00000100:00100000:3.0:1767319846.115146:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319846.115204:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273216 00000100:00100000:0.0:1767319846.115209:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273216 00000100:00100000:0.0:1767319846.115236:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4864 00000100:00100000:0.0:1767319846.115242:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985273216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319846.115372:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985273216:12345-0@lo:400:kworker.0 Request processed in 129us (227us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319846.115378:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4864 00000100:00100000:3.0:1767319846.115388:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985273216:0@lo:400:kworker.0 00000100:00080000:0.0:1767319846.115400:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319846.115424:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319846.115429:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319846.115433:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985273088:0@lo:38:kworker.0 00000100:00080000:0.0:1767319846.115445:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319846.115448:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319846.115453:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985273344:0@lo:38:kworker.0 00000100:00100000:3.0:1767319846.115643:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985272960:0@lo:400:kworker.0 00000100:00100000:3.0:1767319846.115663:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985273600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319846.121588:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273472 00000100:00100000:3.0:1767319846.121602:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273472 00000100:00080000:0.0:1767319846.121635:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319846.121671:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3182 00000100:00100000:3.0:1767319846.121681:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985273472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319846.121770:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985273472:12345-0@lo:400:kworker.0 Request processed in 90us (9325us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319846.121779:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3182 00000100:00100000:0.0:1767319846.121815:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985273472:0@lo:400:kworker.0 00010000:00010000:1.0:1767319846.747370:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319847.643382:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319847.643405:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985273728:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319847.643485:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319847.643732:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273728 00000100:00100000:1.0:1767319847.643741:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273728 00000100:00100000:1.0:1767319847.643797:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2364 00000100:00100000:1.0:1767319847.643808:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985273728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319847.643839:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319847.643891:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985273728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (406us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319847.643898:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2364 00000100:00100000:1.0:1767319847.643949:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985273728:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319847.644352:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985273856:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319847.644384:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319847.644467:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273856 00000100:00100000:2.0:1767319847.644473:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273856 00000100:00100000:2.0:1767319847.644521:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2365 00000100:00100000:2.0:1767319847.644530:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985273856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319847.644547:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319847.644590:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985273856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (205us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319847.644596:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2365 00000100:00100000:0.0:1767319847.644922:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985273856:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319847.771386:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319848.796035:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319849.821061:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319850.843870:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319851.177786:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319851.177874:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920583936 00000100:00100000:0.0:1767319851.177882:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920583936 00000100:00100000:0.0:1767319851.178026:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4865 00000100:00100000:0.0:1767319851.178035:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920583936:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:2.0:1767319851.178114:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319851.178119:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920583936:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (343us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319851.178141:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4865 00000100:00100000:2.0:1767319851.178937:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319851.178973:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920583808 00000100:00100000:0.0:1767319851.178978:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920583808 00000100:00100000:0.0:1767319851.179028:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5423 00000100:00100000:0.0:1767319851.179033:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920583808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319851.179050:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319851.179120:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920583808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 85us (184us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319851.179128:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5423 00000100:00080000:1.0:1767319851.229118:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319851.229130:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319851.229133:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319851.229138:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319851.229143:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319851.229151:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5094 00000100:00080000:1.0:1767319851.229156:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319851.229229:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319851.229273:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4380 x1853163985273984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319851.229348:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985273984:0@lo:38:kworker.0 00000100:00100000:2.0:1767319851.229421:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319851.229811:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319851.229816:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319851.229819:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319851.229824:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5094 00000100:00080000:1.0:1767319851.229826:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319851.229830:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319851.229861:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985274112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319851.230042:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985273984 00000100:00100000:0.0:1767319851.230051:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985273984 00000100:00100000:0.0:1767319851.230102:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5424 00000100:00100000:0.0:1767319851.230109:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985273984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319851.230129:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319851.235542:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985274112:0@lo:38:kworker.0 00000100:00100000:2.0:1767319851.235609:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319851.235671:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985274112 00000100:00100000:3.0:1767319851.235682:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985274112 00000100:00100000:3.0:1767319851.235732:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1846 00000100:00100000:3.0:1767319851.235739:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985274112:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319851.235761:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319851.235808:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985274112:12345-0@lo:38:kworker.0 Request processed in 69us (199us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319851.235815:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1846 00000100:00100000:3.0:1767319851.235937:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985274240:0@lo:400:kworker.0 00000100:00100000:3.0:1767319851.235960:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319851.236025:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985273984:12345-0@lo:38:kworker.0 Request processed in 5914us (6603us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319851.236032:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5424 00000100:00100000:0.0:1767319851.236045:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985274240 00000100:00100000:0.0:1767319851.236048:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985274240 00000100:00100000:0.0:1767319851.236099:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4866 00000100:00100000:0.0:1767319851.236107:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985274240:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319851.236181:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319851.236188:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319851.236191:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985273984:0@lo:38:kworker.0 00000100:00080000:2.0:1767319851.236211:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319851.236213:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319851.236215:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985274112:0@lo:38:kworker.0 00000100:00080000:2.0:1767319851.236245:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319851.237161:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319851.237174:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319851.237177:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319851.237181:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5094 00000100:00080000:1.0:1767319851.237185:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319851.237189:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319851.237211:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9f80 x1853163985274368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319851.237234:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319851.237236:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319851.237238:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319851.237239:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5094 00000100:00080000:1.0:1767319851.237241:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319851.237243:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319851.237252:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985274496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319851.237325:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319851.237380:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985274624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319851.237404:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319851.237409:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985274752:0@lo:400:kworker.0 00000100:00100000:1.0:1767319851.237440:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319851.237468:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985274752 00000100:00100000:1.0:1767319851.237472:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985274752 00000100:00100000:1.0:1767319851.237509:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3183 00000100:00100000:1.0:1767319851.237514:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985274752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319851.237540:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985274752:12345-0@lo:400:kworker.0 Request processed in 27us (99us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319851.237545:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3183 00000100:00100000:1.0:1767319851.237567:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985274752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319851.237627:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985274880:0@lo:400:kworker.0 00000100:00100000:2.0:1767319851.237648:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319851.237669:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985274880 00000100:00100000:2.0:1767319851.237671:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985274880 00000100:00100000:2.0:1767319851.237692:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3184 00000100:00100000:2.0:1767319851.237699:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985274880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319851.237728:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985274880:12345-0@lo:400:kworker.0 Request processed in 29us (78us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319851.237734:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3184 00000100:00080000:2.0:1767319851.237755:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319851.237758:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319851.237774:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985274880:0@lo:400:kworker.0 00000100:00100000:2.0:1767319851.237791:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985274368:0@lo:38:kworker.0 00000100:00100000:2.0:1767319851.237813:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319851.237830:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985274624:0@lo:38:kworker.0 00000100:00100000:2.0:1767319851.237848:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319851.237877:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985274496 00000100:00100000:2.0:1767319851.237881:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985274496 00000100:00100000:2.0:1767319851.237909:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985274368 00000100:00100000:2.0:1767319851.237912:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985274368 00000100:00100000:2.0:1767319851.237925:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985274624 00000100:00100000:2.0:1767319851.237927:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985274624 00000100:00100000:2.0:1767319851.237937:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4867 00000100:00100000:2.0:1767319851.237943:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985274496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319851.237976:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985274496:12345-0@lo:400:kworker.0 Request processed in 33us (650us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319851.237982:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4867 00000100:00100000:2.0:1767319851.237994:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5425 00000100:00100000:2.0:1767319851.237997:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985274368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319851.238014:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319851.238039:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985274368:12345-0@lo:38:kworker.0 Request processed in 40us (225us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319851.238045:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5425 00000100:00100000:2.0:1767319851.238056:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5426 00000100:00100000:2.0:1767319851.238060:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985274624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319851.238068:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319851.238101:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985274624:12345-0@lo:38:kworker.0 Request processed in 40us (251us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319851.238107:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5426 00000100:00080000:2.0:1767319851.238128:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319851.238144:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985274496:0@lo:400:kworker.0 00000100:00080000:2.0:1767319851.238184:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319851.238187:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319851.238189:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985274368:0@lo:38:kworker.0 00000100:00080000:2.0:1767319851.238196:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319851.238199:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319851.238200:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985274624:0@lo:38:kworker.0 00000100:00100000:0.0:1767319851.240744:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985274240:12345-0@lo:400:kworker.0 Request processed in 4628us (4772us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319851.240770:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4866 00000100:00100000:3.0:1767319851.246561:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985274240:0@lo:400:kworker.0 00010000:00010000:2.0:1767319851.867388:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319852.763498:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319852.763521:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985275008:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319852.763587:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319852.763836:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985275136:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319852.763866:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319852.764000:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275008 00000100:00100000:2.0:1767319852.764008:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275008 00000100:00100000:2.0:1767319852.764063:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275136 00000100:00100000:2.0:1767319852.764066:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275136 00000100:00100000:2.0:1767319852.764083:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2366 00000100:00100000:2.0:1767319852.764092:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985275008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319852.764121:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319852.764189:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985275008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (578us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319852.764200:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2366 00000100:00100000:2.0:1767319852.764218:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2367 00000100:00100000:2.0:1767319852.764224:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985275136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319852.764240:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319852.764269:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985275136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (402us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319852.764276:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2367 00000100:00100000:2.0:1767319852.764316:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985275136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319852.764346:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985275008:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319852.893964:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319853.915497:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319854.939482:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319855.964521:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319856.314709:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319856.314745:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319856.314776:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920584192 00000100:00100000:0.0:1767319856.314787:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920584192 00000100:00100000:0.0:1767319856.314861:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920584064 00000100:00100000:0.0:1767319856.314864:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920584064 00000100:00100000:0.0:1767319856.314884:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4868 00000100:00100000:0.0:1767319856.314895:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920584192:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319856.314906:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5427 00000100:00100000:2.0:1767319856.314915:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920584064:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319856.314954:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319856.314981:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920584192:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319856.315003:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920584064:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 86us (256us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319856.315016:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5427 00000100:00080000:3.0:1767319856.315057:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319856.315723:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4868 00000100:00080000:3.0:1767319856.350016:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319856.350029:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319856.350032:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319856.350037:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319856.350042:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319856.350050:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5099 00000100:00080000:3.0:1767319856.350056:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319856.350064:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319856.350106:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633b80 x1853163985275264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319856.350143:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319856.350147:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319856.350149:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319856.350152:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5099 00000100:00080000:3.0:1767319856.350154:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319856.350157:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319856.350207:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631c00 x1853163985275392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319856.350250:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319856.350255:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319856.350256:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319856.350260:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5099 00000100:00080000:3.0:1767319856.350262:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319856.350265:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319856.350275:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631880 x1853163985275648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319856.350300:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319856.350303:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319856.350305:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319856.350308:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5099 00000100:00080000:3.0:1767319856.350319:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319856.350322:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319856.350331:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631f80 x1853163985275904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319856.350374:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985275776:0@lo:400:kworker.0 00000100:00100000:3.0:1767319856.350451:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319856.350513:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319856.350519:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985276032:0@lo:400:kworker.0 00000100:00100000:3.0:1767319856.350551:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319856.350552:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985275264:0@lo:38:kworker.0 00000100:00100000:3.0:1767319856.350589:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275776 00000100:00100000:2.0:1767319856.350594:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319856.350599:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275776 00000100:00100000:2.0:1767319856.350628:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985275392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319856.350654:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319856.350661:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275264 00000100:00100000:3.0:1767319856.350665:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275264 00000100:00100000:2.0:1767319856.350688:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985275648:0@lo:38:kworker.0 00000100:00100000:3.0:1767319856.350690:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4869 00000100:00100000:3.0:1767319856.350701:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985275776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319856.350717:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319856.350847:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985276160:0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.351136:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985275520:0@lo:400:kworker.0 00000100:00100000:2.0:1767319856.354669:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985275904:0@lo:38:kworker.0 00000100:00100000:2.0:1767319856.354738:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319856.354801:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276032 00000100:00100000:2.0:1767319856.354809:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276032 00000100:00100000:2.0:1767319856.354861:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3185 00000100:00100000:2.0:1767319856.354871:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985276032:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319856.354930:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985276032:12345-0@lo:400:kworker.0 Request processed in 61us (4380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319856.354938:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3185 00000100:00100000:2.0:1767319856.354963:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275648 00000100:00100000:2.0:1767319856.354967:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275648 00000100:00100000:2.0:1767319856.354978:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275904 00000100:00100000:2.0:1767319856.354980:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275904 00000100:00100000:2.0:1767319856.355003:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5428 00000100:00100000:2.0:1767319856.355008:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985275264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319856.355028:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319856.355065:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985275264:12345-0@lo:38:kworker.0 Request processed in 56us (4472us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319856.355072:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5428 00000100:00100000:2.0:1767319856.355086:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5429 00000100:00100000:2.0:1767319856.355089:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985275648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319856.355100:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319856.355123:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985275648:12345-0@lo:38:kworker.0 Request processed in 32us (4407us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319856.355129:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5429 00000100:00100000:2.0:1767319856.355138:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5430 00000100:00100000:2.0:1767319856.355142:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985275904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319856.355221:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319856.355245:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985275904:12345-0@lo:38:kworker.0 Request processed in 101us (507us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319856.355250:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5430 00000100:00100000:2.0:1767319856.355300:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275392 00000100:00100000:2.0:1767319856.355305:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275392 00000100:00100000:2.0:1767319856.355327:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1847 00000100:00100000:2.0:1767319856.355951:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985275392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319856.355967:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319856.356018:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985275392:12345-0@lo:38:kworker.0 Request processed in 685us (5363us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319856.356025:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1847 00000100:00100000:1.0:1767319856.356067:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319856.356084:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985276032:0@lo:400:kworker.0 00000100:00100000:1.0:1767319856.356124:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276160 00000100:00100000:1.0:1767319856.356130:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276160 00000100:00080000:2.0:1767319856.356133:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319856.356140:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319856.356145:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985275264:0@lo:38:kworker.0 00000100:00080000:2.0:1767319856.356158:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319856.356181:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3186 00000100:00080000:2.0:1767319856.356185:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319856.356192:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985276160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319856.356193:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985275392:0@lo:38:kworker.0 00000100:00080000:2.0:1767319856.356210:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319856.356214:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319856.356218:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985275648:0@lo:38:kworker.0 00000100:00080000:2.0:1767319856.356232:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319856.356236:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319856.356240:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985276160:12345-0@lo:400:kworker.0 Request processed in 49us (172us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319856.356243:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985275904:0@lo:38:kworker.0 00000100:00100000:1.0:1767319856.356253:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3186 00000100:00100000:1.0:1767319856.356291:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985276160:0@lo:400:kworker.0 00000100:00080000:2.0:1767319856.356311:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319856.356317:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319856.356320:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319856.358427:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985275776:12345-0@lo:400:kworker.0 Request processed in 7719us (7967us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319856.358441:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4869 00000100:00100000:0.0:1767319856.358654:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319856.359000:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985275520 00000100:00100000:3.0:1767319856.359009:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985275520 00000100:00100000:3.0:1767319856.359076:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4870 00000100:00100000:3.0:1767319856.359083:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985275520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319856.359135:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985275520:12345-0@lo:400:kworker.0 Request processed in 52us (481us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319856.359142:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4870 00000100:00100000:0.0:1767319856.359529:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985275520:0@lo:400:kworker.0 00000100:00080000:0.0:1767319856.359551:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319856.359746:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985275776:0@lo:400:kworker.0 00010000:00010000:1.0:1767319856.987725:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319857.883407:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985276288:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319857.883483:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319857.883663:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985276416:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319857.883688:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319857.883767:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276288 00000100:00100000:1.0:1767319857.883775:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276288 00000100:00100000:1.0:1767319857.883822:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276416 00000100:00100000:1.0:1767319857.883825:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276416 00000100:00100000:1.0:1767319857.883845:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2368 00000100:00100000:1.0:1767319857.883853:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985276288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319857.883919:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319857.883996:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985276288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 144us (512us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319857.884004:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2368 00000100:00100000:1.0:1767319857.884020:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2369 00000100:00100000:1.0:1767319857.884026:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985276416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319857.884040:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319857.884069:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985276416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319857.884074:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2369 00000100:00100000:1.0:1767319857.884115:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985276288:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319857.884146:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985276416:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319858.013493:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319859.035378:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319860.059486:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319861.083702:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319861.420083:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319861.420223:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920584448 00000100:00100000:1.0:1767319861.420233:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920584448 00000100:00100000:3.0:1767319861.420833:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319861.421005:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920584320 00000100:00100000:1.0:1767319861.421012:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920584320 00000100:00100000:1.0:1767319861.421031:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4871 00000100:00100000:1.0:1767319861.421042:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920584448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319861.421116:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920584448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (1043us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319861.421124:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4871 00000100:00100000:1.0:1767319861.421144:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5431 00000100:00100000:1.0:1767319861.421149:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920584320:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319861.421203:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319861.421231:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920584320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 80us (407us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319861.421237:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5431 00000100:00080000:0.0:1767319861.421425:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319861.467962:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319861.467978:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319861.467982:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319861.467987:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319861.467993:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319861.468005:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5104 00000100:00080000:3.0:1767319861.468012:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319861.468021:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319861.468080:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5597800 x1853163985276544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319861.468124:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319861.468128:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319861.468130:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319861.468135:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5104 00000100:00080000:3.0:1767319861.468137:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319861.468141:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319861.468156:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595500 x1853163985276672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319861.468775:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319861.468780:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319861.468782:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319861.469049:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5104 00000100:00080000:3.0:1767319861.469059:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319861.469078:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319861.469128:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594700 x1853163985276928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319861.469222:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319861.469228:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319861.469231:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319861.469234:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5104 00000100:00080000:3.0:1767319861.469237:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319861.469240:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319861.469255:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595f80 x1853163985277184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319861.469304:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985277056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319861.469364:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.469434:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319861.469441:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985277312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319861.469469:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.469501:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985277056 00000100:00100000:3.0:1767319861.469509:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985277056 00000100:00100000:3.0:1767319861.469564:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4872 00000100:00100000:3.0:1767319861.469573:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985277056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319861.469621:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985277056:12345-0@lo:400:kworker.0 Request processed in 48us (256us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319861.469628:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4872 00000100:00100000:3.0:1767319861.471037:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985277056:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.473448:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985276800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.473552:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319861.473648:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319861.473659:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985277440:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.473694:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.473795:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276800 00000100:00100000:3.0:1767319861.473809:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276800 00000100:00100000:3.0:1767319861.473884:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4873 00000100:00100000:3.0:1767319861.473904:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985276800:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319861.473999:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985276800:12345-0@lo:400:kworker.0 Request processed in 98us (447us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319861.474007:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4873 00000100:00100000:2.0:1767319861.474509:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985276800:0@lo:400:kworker.0 00000100:00100000:1.0:1767319861.474980:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985277312 00000100:00100000:1.0:1767319861.474989:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985277312 00000100:00100000:1.0:1767319861.475053:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985277440 00000100:00100000:1.0:1767319861.475056:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985277440 00000100:00100000:1.0:1767319861.475280:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3187 00000100:00100000:1.0:1767319861.475297:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985277312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319861.475383:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985277312:12345-0@lo:400:kworker.0 Request processed in 90us (5910us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319861.475385:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3188 00000100:00100000:1.0:1767319861.475412:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3187 00000100:00100000:2.0:1767319861.475414:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985277440:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319861.475499:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985277312:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.476795:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985277440:12345-0@lo:400:kworker.0 Request processed in 1384us (3092us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319861.477502:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3188 00000100:00100000:3.0:1767319861.477690:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985277440:0@lo:400:kworker.0 00000100:00080000:0.0:1767319861.488992:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319861.489011:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319861.489017:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319861.490439:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985276544:0@lo:38:kworker.0 00000100:00100000:0.0:1767319861.493549:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319861.493646:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276544 00000100:00100000:1.0:1767319861.493664:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276544 00000100:00100000:0.0:1767319861.493680:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985276672:0@lo:38:kworker.0 00000100:00100000:1.0:1767319861.493737:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5432 00000100:00100000:0.0:1767319861.493747:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319861.493753:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985276544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319861.493781:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319861.493788:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985276928:0@lo:38:kworker.0 00000100:00100000:0.0:1767319861.493823:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319861.493844:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985277184:0@lo:38:kworker.0 00000100:00100000:2.0:1767319861.493964:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276672 00000100:00100000:2.0:1767319861.493973:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276672 00000100:00100000:2.0:1767319861.494014:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1848 00000100:00100000:2.0:1767319861.494022:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985276672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319861.494040:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319861.494273:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985276928 00000100:00100000:3.0:1767319861.494278:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985276928 00000100:00100000:3.0:1767319861.494310:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5433 00000100:00100000:3.0:1767319861.494316:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985276928:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319861.494326:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319861.495131:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319861.495268:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985277184 00000100:00100000:0.0:1767319861.495275:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985277184 00000100:00100000:0.0:1767319861.495306:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5434 00000100:00100000:0.0:1767319861.495313:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985277184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319861.495334:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319861.495377:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985277184:12345-0@lo:38:kworker.0 Request processed in 63us (249us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319861.495384:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5434 00000100:00080000:0.0:1767319861.495431:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319861.495437:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319861.495441:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985277184:0@lo:38:kworker.0 00000100:00100000:1.0:1767319861.501557:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985276544:12345-0@lo:38:kworker.0 Request processed in 7799us (8009us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319861.501572:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5432 00000100:00080000:0.0:1767319861.502411:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319861.502428:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319861.502435:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985276544:0@lo:38:kworker.0 00000100:00100000:2.0:1767319861.505052:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985276672:12345-0@lo:38:kworker.0 Request processed in 11015us (11292us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319861.505075:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1848 00000100:00080000:0.0:1767319861.505338:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319861.505346:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319861.505351:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985276672:0@lo:38:kworker.0 00000100:00100000:3.0:1767319861.505721:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985276928:12345-0@lo:38:kworker.0 Request processed in 11401us (11896us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319861.505730:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5433 00000100:00080000:0.0:1767319861.505793:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319861.505798:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319861.506115:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985276928:0@lo:38:kworker.0 00010000:00010000:1.0:1767319862.109302:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319863.004229:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319863.004251:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985277568:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319863.004360:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319863.005066:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985277568 00000100:00100000:2.0:1767319863.005078:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985277568 00000100:00100000:0.0:1767319863.005101:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319863.005119:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985277696:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319863.005161:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2370 00000100:00100000:0.0:1767319863.005203:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319863.005220:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985277568:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319863.005244:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319863.005261:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985277696 00000100:00100000:0.0:1767319863.005265:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985277696 00002000:00000024:2.0:1767319863.005266:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319863.005516:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985277568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 301us (1156us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319863.005525:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2370 00000100:00100000:2.0:1767319863.005559:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2371 00000100:00100000:2.0:1767319863.005566:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985277696:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319863.005585:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319863.005620:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985277696:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (417us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319863.005627:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2371 00000100:00100000:2.0:1767319863.005687:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985277568:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319863.005825:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319863.005852:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985277696:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319863.131761:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319864.155878:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319865.179534:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319865.514368:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767319865.514416:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920584576 offset 0 length 520: 4 00000100:00100000:2.0:1767319865.514447:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319865.514567:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920584832 00000100:00100000:1.0:1767319865.514581:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920584832 00000100:00100000:1.0:1767319865.514736:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920584704 00000100:00100000:1.0:1767319865.514741:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920584704 00000100:00100000:1.0:1767319865.514770:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4874 00000100:00100000:1.0:1767319865.514782:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920584832:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319865.514878:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5435 00000100:00100000:3.0:1767319865.514888:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920584704:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767319865.514901:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920584832:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 120us (546us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319865.514910:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4874 00010000:00020000:3.0:1767319865.514917:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319865.515525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319865.516799:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920584704:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1899us (2338us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319865.516823:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5435 00000100:00100000:2.0:1767319865.627319:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319840/real 1767319840] req@ffff8ce1b2dc1f80 x1853163985271296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319865 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319865.627356:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b2dc1f80 x1853163985271296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319865.627377:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319865.627469:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319865.627474:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985271296:0@lo:250:kworker.0 00000100:00080000:1.0:1767319865.629277:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319865.629287:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319865.629354:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319865.629363:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5089 00000100:00080000:1.0:1767319865.629368:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319865.629376:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319865.629415:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985277824/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319865.629511:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985277824:0@lo:250:kworker.0 00000400:00000100:2.0:1767319865.629574:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985277824 offset 0 length 520: 4 00000100:00080000:1.0:1767319865.629641:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319865.629649:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319865.629651:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319865.629655:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5109 00000100:00080000:1.0:1767319865.629657:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319865.629661:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319865.629694:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9f80 x1853163985277952/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319865.632631:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985277952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319865.634047:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319865.634635:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319865.634642:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319865.634645:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319865.634652:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5109 00000100:00080000:1.0:1767319865.634657:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319865.634664:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319865.634786:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9180 x1853163985278080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319865.634838:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319865.634841:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319865.634843:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319865.634847:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5109 00000100:00080000:1.0:1767319865.634849:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319865.634852:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319865.634864:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985278336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319865.634889:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319865.634893:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319865.634895:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319865.634898:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5109 00000100:00080000:1.0:1767319865.634900:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319865.634903:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319865.634914:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af100 x1853163985278592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319865.634953:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985278848:0@lo:400:kworker.0 00000100:00100000:1.0:1767319865.634999:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319865.635028:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319865.635035:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985278208:0@lo:400:kworker.0 00000100:00100000:1.0:1767319865.635056:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319865.635085:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985278464:0@lo:400:kworker.0 00000100:00100000:1.0:1767319865.635106:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319865.635273:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278848 00000100:00100000:0.0:1767319865.635283:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278848 00000100:00100000:0.0:1767319865.635357:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3189 00000100:00100000:0.0:1767319865.635366:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985278848:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319865.635418:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985278848:12345-0@lo:400:kworker.0 Request processed in 53us (417us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319865.635425:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3189 00000100:00080000:0.0:1767319865.635446:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319865.635462:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985277952 00000100:00100000:0.0:1767319865.635466:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985277952 00000100:00100000:0.0:1767319865.635484:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278208 00000100:00100000:0.0:1767319865.635486:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278208 00000100:00100000:0.0:1767319865.635510:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278464 00000100:00100000:0.0:1767319865.635514:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278464 00000100:00100000:0.0:1767319865.635533:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4875 00000100:00100000:0.0:1767319865.635540:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985278208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319865.635568:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985278208:12345-0@lo:400:kworker.0 Request processed in 30us (511us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319865.635575:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4875 00000100:00100000:0.0:1767319865.635586:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4876 00000100:00100000:0.0:1767319865.635591:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985278464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319865.635613:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985278464:12345-0@lo:400:kworker.0 Request processed in 22us (506us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319865.635618:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4876 00000100:00100000:0.0:1767319865.635630:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5436 00000100:00100000:0.0:1767319865.635634:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985277952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319865.635653:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319865.635692:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985277952:12345-0@lo:38:kworker.0 Request processed in 57us (1651us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319865.635698:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5436 00000100:00080000:0.0:1767319865.635716:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319865.635765:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985278208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319865.635776:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985278848:0@lo:400:kworker.0 00000100:00100000:2.0:1767319865.636104:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985278080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319865.636148:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319865.636199:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985278336:0@lo:38:kworker.0 00000100:00100000:2.0:1767319865.636221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319865.636237:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985278592:0@lo:38:kworker.0 00000100:00100000:2.0:1767319865.636258:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319865.636281:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319865.636286:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319865.636461:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985277952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319865.636496:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278336 00000100:00100000:2.0:1767319865.636502:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278336 00000100:00100000:2.0:1767319865.636522:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278592 00000100:00100000:2.0:1767319865.636524:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278592 00000100:00100000:2.0:1767319865.636537:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5437 00000100:00100000:2.0:1767319865.636542:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985278336:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319865.636557:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319865.636586:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985278336:12345-0@lo:38:kworker.0 Request processed in 43us (364us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319865.636596:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5437 00000100:00100000:2.0:1767319865.636612:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5438 00000100:00100000:2.0:1767319865.636615:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985278592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319865.636625:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319865.636647:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985278592:12345-0@lo:38:kworker.0 Request processed in 30us (388us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319865.636653:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5438 00000100:00100000:3.0:1767319865.636823:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985278720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319865.636857:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319865.636889:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278720 00000100:00100000:3.0:1767319865.636893:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278720 00000100:00100000:3.0:1767319865.636932:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3190 00000100:00100000:3.0:1767319865.636938:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985278720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319865.636974:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985278720:12345-0@lo:400:kworker.0 Request processed in 37us (117us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319865.636984:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3190 00000100:00080000:3.0:1767319865.636999:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319865.637017:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985278720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319865.637096:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985278464:0@lo:400:kworker.0 00000100:00100000:0.0:1767319865.637234:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278080 00000100:00100000:0.0:1767319865.637239:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278080 00000100:00100000:0.0:1767319865.637262:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1849 00000100:00100000:0.0:1767319865.637266:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985278080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319865.637277:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319865.637663:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985278080:12345-0@lo:38:kworker.0 Request processed in 395us (1513us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319865.637681:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1849 00000100:00080000:3.0:1767319865.637825:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319865.637831:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319865.637834:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985278080:0@lo:38:kworker.0 00000100:00080000:3.0:1767319865.637844:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319865.637847:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319865.637850:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985278336:0@lo:38:kworker.0 00000100:00080000:3.0:1767319865.637857:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319865.637861:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319865.637863:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985278592:0@lo:38:kworker.0 00010000:00010000:0.0:1767319866.203453:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319867.230237:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319868.123627:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319868.123653:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985278976:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319868.123731:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319868.123908:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985278976 00000100:00100000:2.0:1767319868.123916:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985278976 00000100:00100000:2.0:1767319868.123969:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2372 00000100:00100000:2.0:1767319868.123976:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985278976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319868.124008:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319868.124041:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985278976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (311us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319868.124047:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2372 00000100:00080000:3.0:1767319868.124192:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319868.124221:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985278976:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319868.124804:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985279104:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319868.124879:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319868.125196:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985279104 00000100:00100000:1.0:1767319868.125201:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985279104 00000100:00100000:1.0:1767319868.125240:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2373 00000100:00100000:1.0:1767319868.125245:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985279104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319868.125262:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319868.125300:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985279104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (420us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319868.125305:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2373 00000100:00080000:3.0:1767319868.125369:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319868.125410:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985279104:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319868.251646:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319869.278822:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319870.299382:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319870.635049:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319870.635354:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319870.636080:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920584960 00000100:00100000:3.0:1767319870.636092:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920584960 00000100:00100000:0.0:1767319870.636271:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920585088 00000100:00100000:0.0:1767319870.636277:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920585088 00000100:00100000:3.0:1767319870.636295:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5439 00000100:00100000:3.0:1767319870.636305:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920584960:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319870.636327:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319870.636387:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920584960:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 83us (1348us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319870.636408:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5439 00000100:00080000:2.0:1767319870.636551:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319870.637239:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4877 00000100:00100000:1.0:1767319870.637251:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920585088:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319870.637359:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920585088:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 111us (2005us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319870.637367:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4877 00000100:00080000:1.0:1767319870.683287:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.683297:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319870.683300:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.683304:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.683308:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.683316:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5113 00000100:00080000:1.0:1767319870.683320:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.683327:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319870.683368:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae680 x1853163985279232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319870.683396:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.683399:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.683401:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.683404:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5113 00000100:00080000:1.0:1767319870.683406:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.683410:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319870.683424:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad880 x1853163985279360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319870.683541:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319870.683681:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985279232:0@lo:38:kworker.0 00000100:00100000:3.0:1767319870.683744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319870.683792:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985279360:0@lo:38:kworker.0 00000100:00100000:3.0:1767319870.683810:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319870.683860:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985279360 00000100:00100000:3.0:1767319870.683867:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985279360 00000100:00100000:3.0:1767319870.683901:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1850 00000100:00100000:3.0:1767319870.683908:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985279360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319870.683929:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319870.683971:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985279360:12345-0@lo:38:kworker.0 Request processed in 63us (160us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319870.683978:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1850 00000100:00080000:3.0:1767319870.684031:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319870.684035:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319870.684039:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985279360:0@lo:38:kworker.0 00000100:00080000:1.0:1767319870.684092:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.684096:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.684098:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.684103:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5113 00000100:00080000:1.0:1767319870.684105:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.684109:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319870.684123:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac000 x1853163985279616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319870.684161:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.684183:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.684185:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.684189:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5113 00000100:00080000:1.0:1767319870.684191:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.684194:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319870.684205:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adf80 x1853163985279872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319870.684256:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985279232 00000100:00100000:2.0:1767319870.684259:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985279232 00000100:00100000:2.0:1767319870.684279:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5440 00000100:00100000:2.0:1767319870.684282:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985279232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319870.684290:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319870.684312:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985279232:12345-0@lo:38:kworker.0 Request processed in 29us (568us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319870.684315:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5440 00000100:00100000:2.0:1767319870.684344:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985280000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319870.684364:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319870.684376:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319870.684378:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985279744:0@lo:400:kworker.0 00000100:00100000:2.0:1767319870.684407:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319870.684423:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985279744 00000100:00100000:2.0:1767319870.684426:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985279744 00000100:00100000:2.0:1767319870.684451:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4878 00000100:00100000:2.0:1767319870.684456:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985279744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319870.684493:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985279744:12345-0@lo:400:kworker.0 Request processed in 37us (84us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319870.684498:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4878 00000100:00080000:2.0:1767319870.684513:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319870.684589:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985279744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319870.684650:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280000 00000100:00100000:3.0:1767319870.684654:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280000 00000100:00100000:3.0:1767319870.684690:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3191 00000100:00100000:3.0:1767319870.684696:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985280000:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319870.684736:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985280000:12345-0@lo:400:kworker.0 Request processed in 40us (370us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319870.684742:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3191 00000100:00100000:3.0:1767319870.684765:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985280000:0@lo:400:kworker.0 00000100:00100000:3.0:1767319870.684781:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985279616:0@lo:38:kworker.0 00000100:00100000:3.0:1767319870.684802:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319870.684814:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985279872:0@lo:38:kworker.0 00000100:00100000:3.0:1767319870.684831:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319870.684848:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319870.684852:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319870.684855:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985279232:0@lo:38:kworker.0 00000100:00100000:3.0:1767319870.684874:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985279616 00000100:00100000:3.0:1767319870.684878:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985279616 00000100:00100000:3.0:1767319870.684892:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985279872 00000100:00100000:3.0:1767319870.684894:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985279872 00000100:00100000:3.0:1767319870.684908:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5441 00000100:00100000:3.0:1767319870.684912:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985279616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319870.684922:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319870.684947:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985279616:12345-0@lo:38:kworker.0 Request processed in 34us (144us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319870.684953:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5441 00000100:00100000:3.0:1767319870.684964:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5442 00000100:00100000:3.0:1767319870.684968:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985279872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319870.684978:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319870.684997:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985279872:12345-0@lo:38:kworker.0 Request processed in 28us (166us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319870.685003:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5442 00000100:00080000:3.0:1767319870.685028:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319870.685033:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319870.685035:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985279616:0@lo:38:kworker.0 00000100:00080000:3.0:1767319870.685049:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319870.685052:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319870.685054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985279872:0@lo:38:kworker.0 00000100:00080000:2.0:1767319870.685206:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319870.690787:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985280128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.690889:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.690938:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985279488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.690965:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.691252:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985279488 00000100:00100000:0.0:1767319870.691261:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985279488 00000100:00100000:0.0:1767319870.691330:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4879 00000100:00100000:0.0:1767319870.691339:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985279488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.691389:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985279488:12345-0@lo:400:kworker.0 Request processed in 51us (423us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319870.691396:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4879 00000100:00080000:0.0:1767319870.691415:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319870.691445:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985279488:0@lo:400:kworker.0 00000100:00100000:3.0:1767319870.691545:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280128 00000100:00100000:3.0:1767319870.691549:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280128 00000100:00100000:3.0:1767319870.691590:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3192 00000100:00100000:3.0:1767319870.691597:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985280128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319870.691629:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985280128:12345-0@lo:400:kworker.0 Request processed in 34us (741us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319870.691635:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3192 00000100:00080000:0.0:1767319870.691880:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319870.691899:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985280128:0@lo:400:kworker.0 00010000:00010000:0.0:1767319871.324983:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319872.347412:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319873.243510:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985280256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319873.243585:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319873.243757:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280256 00000100:00100000:2.0:1767319873.243766:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280256 00000100:00100000:2.0:1767319873.243837:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2374 00000100:00100000:2.0:1767319873.243848:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985280256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319873.243876:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319873.243917:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985280256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (332us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319873.243924:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2374 00000100:00080000:0.0:1767319873.244059:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319873.244273:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985280256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319873.244638:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985280384:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319873.244682:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319873.245044:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280384 00000100:00100000:1.0:1767319873.245050:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280384 00000100:00100000:1.0:1767319873.245104:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2375 00000100:00100000:1.0:1767319873.245112:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985280384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319873.245132:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319873.245206:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985280384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (523us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319873.245213:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2375 00000100:00100000:1.0:1767319873.245246:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985280384:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319873.245358:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319873.371397:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319874.396698:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319875.420405:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319875.762795:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319875.762920:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319875.763200:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920585344 00000100:00100000:1.0:1767319875.763210:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920585344 00000100:00100000:1.0:1767319875.763289:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920585216 00000100:00100000:1.0:1767319875.763292:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920585216 00000100:00100000:1.0:1767319875.763310:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4880 00000100:00100000:1.0:1767319875.763329:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920585344:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319875.763402:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920585344:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (617us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319875.763409:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4880 00000100:00100000:1.0:1767319875.763426:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5443 00000100:00100000:1.0:1767319875.763430:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920585216:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319875.763450:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319875.763474:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920585216:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (555us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319875.763480:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5443 00000100:00080000:2.0:1767319875.764161:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319875.804696:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319875.804707:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319875.804709:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319875.804712:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319875.804716:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319875.804724:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5118 00000100:00080000:1.0:1767319875.804729:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319875.804736:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319875.804775:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad180 x1853163985280512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319875.804801:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319875.804805:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319875.804806:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319875.804809:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5118 00000100:00080000:1.0:1767319875.804811:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319875.804814:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319875.804828:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985280640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319875.804871:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319875.804875:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319875.804877:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319875.804880:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5118 00000100:00080000:1.0:1767319875.804882:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319875.804885:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319875.804895:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985280896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319875.804920:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319875.804922:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319875.804924:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319875.804926:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5118 00000100:00080000:1.0:1767319875.804929:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319875.804931:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319875.804941:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adf80 x1853163985281152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319875.804978:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985281280:0@lo:400:kworker.0 00000100:00100000:1.0:1767319875.805039:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319875.805043:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985280768:0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.805073:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319875.805097:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319875.805097:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319875.805103:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985281024:0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.805103:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985281408:0@lo:400:kworker.0 00000100:00100000:1.0:1767319875.805122:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319875.805122:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319875.805149:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280768 00000100:00100000:2.0:1767319875.805149:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281280 00000100:00100000:1.0:1767319875.805156:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280768 00000100:00100000:2.0:1767319875.805156:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281280 00000100:00100000:1.0:1767319875.805214:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281024 00000100:00100000:1.0:1767319875.805218:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281024 00000100:00100000:3.0:1767319875.805223:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985280512:0@lo:38:kworker.0 00000100:00100000:0.0:1767319875.805281:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4881 00000100:00100000:0.0:1767319875.805291:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985280768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319875.805342:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985280768:12345-0@lo:400:kworker.0 Request processed in 53us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319875.805349:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4881 00000100:00100000:0.0:1767319875.805396:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985280768:0@lo:400:kworker.0 00000100:00100000:0.0:1767319875.805428:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281408 00000100:00100000:0.0:1767319875.805432:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281408 00000100:00100000:3.0:1767319875.805481:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319875.805504:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985280640:0@lo:38:kworker.0 00000100:00100000:3.0:1767319875.805796:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319875.805822:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985280896:0@lo:38:kworker.0 00000100:00100000:3.0:1767319875.805842:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319875.805857:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985281152:0@lo:38:kworker.0 00000100:00100000:3.0:1767319875.805875:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319875.805908:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280640 00000100:00100000:3.0:1767319875.805912:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280640 00000100:00100000:3.0:1767319875.805934:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1851 00000100:00100000:3.0:1767319875.805939:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985280640:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319875.805968:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319875.806002:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985280640:12345-0@lo:38:kworker.0 Request processed in 61us (205us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319875.806007:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1851 00000100:00080000:3.0:1767319875.806036:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319875.806040:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319875.806043:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985280640:0@lo:38:kworker.0 00000100:00100000:3.0:1767319875.806069:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280512 00000100:00100000:3.0:1767319875.806072:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280512 00000100:00100000:3.0:1767319875.806091:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985280896 00000100:00100000:3.0:1767319875.806096:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985280896 00000100:00100000:3.0:1767319875.806103:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281152 00000100:00100000:3.0:1767319875.806104:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281152 00000100:00100000:3.0:1767319875.806112:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5444 00000100:00100000:3.0:1767319875.806116:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985280512:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319875.806125:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319875.806152:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985280512:12345-0@lo:38:kworker.0 Request processed in 35us (670us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319875.806157:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5444 00000100:00100000:3.0:1767319875.806187:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5445 00000100:00100000:3.0:1767319875.806191:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985280896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319875.806198:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319875.806216:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985280896:12345-0@lo:38:kworker.0 Request processed in 24us (374us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319875.806222:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5445 00000100:00100000:3.0:1767319875.806231:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5446 00000100:00100000:3.0:1767319875.806235:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985281152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319875.806243:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319875.806264:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985281152:12345-0@lo:38:kworker.0 Request processed in 27us (388us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319875.806269:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5446 00000100:00080000:3.0:1767319875.806290:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319875.806294:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319875.807456:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985280512:0@lo:38:kworker.0 00000100:00080000:3.0:1767319875.807482:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319875.807487:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319875.807491:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985280896:0@lo:38:kworker.0 00000100:00080000:3.0:1767319875.807500:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319875.807503:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319875.807506:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985281152:0@lo:38:kworker.0 00000100:00100000:1.0:1767319875.808872:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4882 00000100:00100000:1.0:1767319875.808885:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985281024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319875.808987:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985281024:12345-0@lo:400:kworker.0 Request processed in 104us (3862us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319875.808994:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4882 00000100:00100000:1.0:1767319875.809042:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985281024:0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.813027:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3193 00000100:00100000:2.0:1767319875.813047:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985281280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.813142:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985281280:12345-0@lo:400:kworker.0 Request processed in 103us (8102us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319875.813149:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3193 00000100:00100000:2.0:1767319875.813237:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985281280:0@lo:400:kworker.0 00000100:00100000:0.0:1767319875.813543:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3194 00000100:00100000:0.0:1767319875.813553:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985281408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319875.813592:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985281408:12345-0@lo:400:kworker.0 Request processed in 42us (8469us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319875.813598:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3194 00000100:00100000:0.0:1767319875.813655:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985281408:0@lo:400:kworker.0 00000100:00080000:3.0:1767319875.813820:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319875.813825:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319875.813829:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319876.443898:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319877.467366:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319878.363544:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985281664:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319878.363649:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319878.363923:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281664 00000100:00100000:3.0:1767319878.363932:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281664 00000100:00100000:3.0:1767319878.364000:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2376 00000100:00100000:3.0:1767319878.364021:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985281664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319878.364053:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319878.364101:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985281664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (451us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319878.364108:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2376 00000100:00100000:3.0:1767319878.364155:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985281664:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319878.364259:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985281536:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319878.364290:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319878.364800:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281536 00000100:00100000:3.0:1767319878.364806:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281536 00000100:00100000:3.0:1767319878.364852:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2377 00000100:00100000:3.0:1767319878.364859:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985281536:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319878.364879:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319878.364918:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985281536:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (626us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319878.364924:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2377 00000100:00100000:3.0:1767319878.364957:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985281536:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319878.365223:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319878.365227:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767319878.492112:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319879.515410:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319880.539919:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319880.875007:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319880.875280:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920585600 00000100:00100000:1.0:1767319880.875292:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920585600 00000100:00100000:1.0:1767319880.875379:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4883 00000100:00100000:1.0:1767319880.875390:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920585600:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319880.875473:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920585600:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (477us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319880.875481:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4883 00000100:00080000:3.0:1767319880.875758:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319880.877815:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319880.877918:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920585472 00000100:00100000:1.0:1767319880.877930:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920585472 00000100:00100000:1.0:1767319880.877975:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5447 00000100:00100000:1.0:1767319880.877982:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920585472:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319880.878001:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319880.878092:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920585472:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 109us (286us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319880.878100:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5447 00000100:00080000:3.0:1767319880.926738:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.926747:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319880.926751:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.926755:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.926760:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.926769:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.926773:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.926780:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319880.926995:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5597100 x1853163985281792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319880.927198:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985281792:0@lo:38:kworker.0 00000100:00100000:0.0:1767319880.927262:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319880.927330:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281792 00000100:00100000:1.0:1767319880.927340:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281792 00000100:00100000:1.0:1767319880.927388:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5448 00000100:00100000:1.0:1767319880.927397:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985281792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319880.927417:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319880.927421:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.927426:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.927428:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.927432:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.927435:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.927439:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319880.927456:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595c00 x1853163985281920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319880.927641:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985281920:0@lo:38:kworker.0 00000100:00100000:0.0:1767319880.927672:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319880.927744:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985281920 00000100:00100000:2.0:1767319880.927749:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985281920 00000100:00100000:2.0:1767319880.927771:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1852 00000100:00100000:2.0:1767319880.927776:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985281920:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319880.927788:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319880.927838:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985281920:12345-0@lo:38:kworker.0 Request processed in 60us (165us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319880.927845:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1852 00000100:00100000:1.0:1767319880.930096:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985281792:12345-0@lo:38:kworker.0 Request processed in 2688us (2822us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319880.930109:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5448 00000100:00080000:0.0:1767319880.930408:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.930414:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.930418:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985281792:0@lo:38:kworker.0 00000100:00080000:0.0:1767319880.930446:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.930450:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.930453:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985281920:0@lo:38:kworker.0 00000100:00080000:3.0:1767319880.930615:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.930620:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.930622:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.930627:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.930630:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.930636:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319880.930655:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5597100 x1853163985282176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319880.930710:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.930714:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.930716:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.930719:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.930721:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.930725:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319880.930736:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc000 x1853163985282432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319880.930774:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985282304:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.930993:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985282048:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.931028:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319880.931056:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319880.931062:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985282688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.931085:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319880.931137:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985282560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.931188:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319880.931218:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282688 00000100:00100000:2.0:1767319880.931224:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282688 00000100:00100000:2.0:1767319880.931271:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282560 00000100:00100000:2.0:1767319880.931274:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282560 00000100:00100000:2.0:1767319880.931302:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3195 00000100:00100000:1.0:1767319880.931309:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3196 00000100:00100000:2.0:1767319880.931314:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985282688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319880.931321:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985282560:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.931372:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985282688:12345-0@lo:400:kworker.0 Request processed in 61us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319880.931382:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985282560:12345-0@lo:400:kworker.0 Request processed in 62us (191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319880.931382:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3195 00000100:00100000:1.0:1767319880.931396:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3196 00000100:00080000:2.0:1767319880.931409:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319880.931414:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319880.931433:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985282560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.931441:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985282688:0@lo:400:kworker.0 00000100:00100000:1.0:1767319880.931454:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282048 00000100:00100000:1.0:1767319880.931458:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282048 00000100:00080000:2.0:1767319880.931475:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319880.931493:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4884 00000100:00100000:1.0:1767319880.931501:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985282048:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319880.931557:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985282048:12345-0@lo:400:kworker.0 Request processed in 57us (528us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319880.931567:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4884 00000100:00100000:3.0:1767319880.931670:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319880.931757:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985282176:0@lo:38:kworker.0 00000100:00100000:0.0:1767319880.931788:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319880.931826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985282432:0@lo:38:kworker.0 00000100:00100000:0.0:1767319880.931853:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319880.932136:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282304 00000100:00100000:3.0:1767319880.932205:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282304 00000100:00100000:3.0:1767319880.932248:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282176 00000100:00100000:3.0:1767319880.932252:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282176 00000100:00100000:1.0:1767319880.932260:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282432 00000100:00100000:1.0:1767319880.932266:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282432 00000100:00100000:3.0:1767319880.932272:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4885 00000100:00100000:3.0:1767319880.932281:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985282304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319880.932296:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5449 00000100:00100000:1.0:1767319880.932303:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985282176:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319880.932321:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985282304:12345-0@lo:400:kworker.0 Request processed in 41us (650us total) trans 0 rc 0/0 00010000:00020000:1.0:1767319880.932323:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319880.932333:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4885 00000100:00100000:3.0:1767319880.932350:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5450 00000100:00100000:3.0:1767319880.932355:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985282432:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319880.932364:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985282176:12345-0@lo:38:kworker.0 Request processed in 59us (573us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767319880.932367:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319880.932374:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5449 00000100:00100000:3.0:1767319880.932401:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985282432:12345-0@lo:38:kworker.0 Request processed in 44us (547us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319880.932408:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5450 00000100:00100000:3.0:1767319880.932440:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985282304:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.932488:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985282048:0@lo:400:kworker.0 00000100:00080000:0.0:1767319880.932509:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319880.932545:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.932551:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.932554:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985282176:0@lo:38:kworker.0 00000100:00080000:0.0:1767319880.932562:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.932565:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.932567:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985282432:0@lo:38:kworker.0 00010000:00010000:1.0:1767319881.564807:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319882.587667:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319883.483557:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319883.483582:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985282944:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319883.483648:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319883.483698:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282944 00000100:00100000:1.0:1767319883.483705:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282944 00000100:00100000:1.0:1767319883.483760:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2378 00000100:00100000:1.0:1767319883.483770:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985282944:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319883.483799:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319883.483843:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985282944:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (194us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319883.483852:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2378 00000100:00080000:1.0:1767319883.483872:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319883.483902:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985282944:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319883.484034:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985282816:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319883.484066:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319883.484162:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985282816 00000100:00100000:3.0:1767319883.484188:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985282816 00000100:00100000:3.0:1767319883.484224:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2379 00000100:00100000:3.0:1767319883.484231:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985282816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319883.484247:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319883.484283:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985282816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (216us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319883.484292:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2379 00000100:00100000:3.0:1767319883.484325:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985282816:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319883.484385:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767319883.611464:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319884.635498:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319885.660539:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319885.994759:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319885.994839:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920585856 00000100:00100000:3.0:1767319885.994850:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920585856 00000100:00100000:3.0:1767319885.994936:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4886 00000100:00100000:3.0:1767319885.994947:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920585856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319885.995027:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920585856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (279us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319885.995035:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4886 00000100:00080000:0.0:1767319885.997161:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319886.010893:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319886.010991:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920585728 00000100:00100000:1.0:1767319886.011001:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920585728 00000100:00100000:1.0:1767319886.011048:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5451 00000100:00100000:1.0:1767319886.011056:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920585728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319886.011078:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319886.011145:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920585728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 88us (260us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319886.011153:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5451 00000100:00080000:3.0:1767319886.043281:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319886.043292:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319886.043295:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319886.043299:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319886.043304:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319886.043312:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5129 00000100:00080000:3.0:1767319886.043318:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319886.043324:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319886.043363:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff480 x1853163985283072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319886.043401:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319886.043405:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319886.043406:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319886.043409:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5129 00000100:00080000:3.0:1767319886.043443:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319886.043446:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319886.043463:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985283200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319886.043505:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985283072:0@lo:38:kworker.0 00000100:00080000:3.0:1767319886.043514:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319886.043519:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319886.043521:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319886.043526:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5129 00000100:00080000:3.0:1767319886.043528:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319886.043532:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319886.043547:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffe300 x1853163985283456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319886.043573:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319886.043579:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319886.043583:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319886.043585:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319886.043590:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5129 00000100:00080000:3.0:1767319886.043592:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319886.043596:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319886.043611:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffea00 x1853163985283712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319886.043648:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985283200:0@lo:38:kworker.0 00000100:00100000:3.0:1767319886.043654:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985283584:0@lo:400:kworker.0 00000100:00100000:0.0:1767319886.043686:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319886.043698:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319886.043722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985283456:0@lo:38:kworker.0 00000100:00100000:0.0:1767319886.043752:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319886.043770:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985283712:0@lo:38:kworker.0 00000100:00100000:0.0:1767319886.043789:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319886.043821:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283200 00000100:00100000:0.0:1767319886.043828:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283200 00000100:00100000:0.0:1767319886.043860:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1853 00000100:00100000:0.0:1767319886.043868:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985283200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319886.043888:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319886.043904:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985283328:0@lo:400:kworker.0 00000100:00100000:0.0:1767319886.043931:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985283200:12345-0@lo:38:kworker.0 Request processed in 61us (243us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319886.043941:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1853 00000100:00100000:1.0:1767319886.043944:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319886.043971:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319886.043976:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985283968:0@lo:400:kworker.0 00000100:00100000:1.0:1767319886.043997:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319886.044022:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283072 00000100:00100000:1.0:1767319886.044026:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283072 00000100:00100000:1.0:1767319886.044044:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283584 00000100:00100000:1.0:1767319886.044046:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283584 00000100:00100000:0.0:1767319886.044064:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283456 00000100:00100000:0.0:1767319886.044069:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283456 00000100:00100000:1.0:1767319886.044077:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283712 00000100:00100000:1.0:1767319886.044083:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283712 00000100:00100000:0.0:1767319886.044088:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283328 00000100:00100000:0.0:1767319886.044093:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283328 00000100:00100000:1.0:1767319886.044115:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4887 00000100:00100000:0.0:1767319886.044121:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4888 00000100:00100000:1.0:1767319886.044126:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985283584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319886.044136:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985283328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319886.044205:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985283584:12345-0@lo:400:kworker.0 Request processed in 83us (508us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319886.044212:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985283328:12345-0@lo:400:kworker.0 Request processed in 80us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319886.044213:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4887 00000100:00100000:0.0:1767319886.044224:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4888 00000100:00100000:1.0:1767319886.044234:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5452 00000100:00100000:1.0:1767319886.044241:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985283072:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319886.044246:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5453 00000100:00100000:0.0:1767319886.044252:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985283456:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319886.044256:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767319886.044264:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319886.044286:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985283072:12345-0@lo:38:kworker.0 Request processed in 45us (713us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319886.044294:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5452 00000100:00100000:0.0:1767319886.044300:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985283456:12345-0@lo:38:kworker.0 Request processed in 45us (545us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319886.044308:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5454 00000100:00100000:2.0:1767319886.044396:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283968 00000100:00100000:2.0:1767319886.044401:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283968 00000100:00100000:2.0:1767319886.044430:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3197 00000100:00100000:2.0:1767319886.044448:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985283968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319886.044477:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985283968:12345-0@lo:400:kworker.0 Request processed in 32us (479us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319886.044482:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3197 00000100:00100000:0.0:1767319886.044522:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5453 00000100:00080000:2.0:1767319886.044526:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319886.044532:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319886.044567:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319886.044572:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319886.044576:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985283456:0@lo:38:kworker.0 00000100:00080000:0.0:1767319886.044588:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319886.044591:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319886.044593:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985283072:0@lo:38:kworker.0 00000100:00080000:0.0:1767319886.044602:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319886.044605:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319886.044607:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985283200:0@lo:38:kworker.0 00000100:00100000:0.0:1767319886.044653:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985283968:0@lo:400:kworker.0 00000100:00100000:0.0:1767319886.044663:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985283328:0@lo:400:kworker.0 00000100:00100000:1.0:1767319886.044851:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985283712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319886.044871:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319886.044913:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985283712:12345-0@lo:38:kworker.0 Request processed in 598us (1122us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319886.044920:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5454 00000100:00080000:0.0:1767319886.045221:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319886.045226:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319886.045228:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985283712:0@lo:38:kworker.0 00000100:00100000:3.0:1767319886.045289:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319886.045303:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985283840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319886.045337:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319886.045373:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985283584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319886.046142:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985283840 00000100:00100000:2.0:1767319886.046149:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985283840 00000100:00100000:2.0:1767319886.046235:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3198 00000100:00080000:0.0:1767319886.046237:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319886.046246:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985283840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319886.046302:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985283840:12345-0@lo:400:kworker.0 Request processed in 59us (963us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319886.046309:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3198 00000100:00100000:0.0:1767319886.046340:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985283840:0@lo:400:kworker.0 00010000:00010000:1.0:1767319886.683343:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319887.707438:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319888.603362:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985284224:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319888.603450:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985284096:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319888.603459:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319888.603524:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985284224 00000100:00100000:3.0:1767319888.603530:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985284224 00000100:00080000:2.0:1767319888.603575:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319888.603596:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2380 00000100:00100000:3.0:1767319888.603605:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985284224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319888.603631:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319888.611590:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985284224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 7976us (8120us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319888.611605:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2380 00000100:00100000:1.0:1767319888.612819:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985284224:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319888.613550:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319888.614321:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985284096 00000100:00100000:3.0:1767319888.614402:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985284096 00000100:00100000:3.0:1767319888.614501:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2381 00000100:00100000:3.0:1767319888.614510:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985284096:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319888.614541:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319888.614607:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985284096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (1058us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319888.614613:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2381 00000100:00100000:1.0:1767319888.614795:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985284096:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319888.614834:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319888.732120:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319889.755605:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319890.139381:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319865/real 1767319865] req@ffff8ce1a4df9500 x1853163985277824/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319890 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319890.139420:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a4df9500 x1853163985277824/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319890 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319890.139439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.139450:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319890.139454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985277824:0@lo:250:kworker.0 00000100:00080000:2.0:1767319890.139711:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.139718:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319890.139722:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.139727:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5113 00000100:00080000:2.0:1767319890.139731:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.139738:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319890.139771:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60380 x1853163985284352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319890.139798:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.139802:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.139803:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.139807:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.139809:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.139812:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319890.139830:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60a80 x1853163985284480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319890.139839:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.139843:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.139844:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.139848:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.139850:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.139852:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319890.139863:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62300 x1853163985284608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319890.140044:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985284736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.140107:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319890.140245:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985284736 00000100:00100000:1.0:1767319890.140255:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985284736 00000100:00100000:1.0:1767319890.140324:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4889 00000100:00100000:1.0:1767319890.140333:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985284736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319890.140404:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985284736:12345-0@lo:400:kworker.0 Request processed in 72us (297us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319890.140412:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4889 00000100:00080000:2.0:1767319890.140648:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.140652:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.140654:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.140658:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.140660:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.140664:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319890.140681:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163985284864/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319890.140706:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.140709:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.140711:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.140714:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.140717:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.140720:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319890.140733:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595c00 x1853163985285120/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319890.140804:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985284992:0@lo:400:kworker.0 00000100:00100000:2.0:1767319890.140834:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319890.140852:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319890.140857:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985285248:0@lo:400:kworker.0 00000100:00100000:2.0:1767319890.140880:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319890.141144:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319890.141161:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985285248 00000100:00100000:3.0:1767319890.141180:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985285248 00000100:00100000:3.0:1767319890.141223:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3199 00000100:00100000:3.0:1767319890.141230:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985285248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.141266:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985285248:12345-0@lo:400:kworker.0 Request processed in 37us (384us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319890.141299:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3199 00000100:00080000:3.0:1767319890.141320:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319890.141347:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985284736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.141374:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319890.141380:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985285376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.141403:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319890.141425:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985285376 00000100:00100000:3.0:1767319890.141429:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985285376 00000100:00100000:3.0:1767319890.141453:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3200 00000100:00100000:3.0:1767319890.141458:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985285376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.141488:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985285376:12345-0@lo:400:kworker.0 Request processed in 30us (84us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319890.141494:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3200 00000100:00080000:3.0:1767319890.141506:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319890.141676:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985284992 00000100:00100000:1.0:1767319890.141680:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985284992 00000100:00100000:1.0:1767319890.141712:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4890 00000100:00100000:1.0:1767319890.141718:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985284992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.141756:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985285376:0@lo:400:kworker.0 00000100:00080000:3.0:1767319890.141770:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319890.141847:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985284352:0@lo:250:kworker.0 00000100:00100000:2.0:1767319890.143351:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985285248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319890.143634:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985284992:12345-0@lo:400:kworker.0 Request processed in 1912us (2796us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319890.143646:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4890 00000100:00100000:2.0:1767319890.143824:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985284992:0@lo:400:kworker.0 00000400:00000100:0.0:1767319890.147102:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985284352 offset 0 length 520: 4 00000100:00100000:0.0:1767319890.147159:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985284480:0@lo:38:kworker.0 00000100:00100000:0.0:1767319890.147221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319890.147333:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985284608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319890.147371:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319890.147400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985284864:0@lo:38:kworker.0 00000100:00100000:0.0:1767319890.147424:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319890.147477:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985284608 00000100:00100000:2.0:1767319890.147485:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985284608 00000100:00100000:2.0:1767319890.147547:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1854 00000100:00100000:2.0:1767319890.147556:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985284608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319890.147576:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319890.147616:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985284608:12345-0@lo:38:kworker.0 Request processed in 60us (244us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319890.147624:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1854 00000100:00100000:0.0:1767319890.153226:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985285120:0@lo:38:kworker.0 00000100:00100000:0.0:1767319890.153286:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319890.157319:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985284480 00000100:00100000:1.0:1767319890.157335:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985284480 00000100:00100000:1.0:1767319890.157403:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985284864 00000100:00100000:1.0:1767319890.157406:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985284864 00000100:00100000:1.0:1767319890.157435:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5455 00000100:00100000:1.0:1767319890.157444:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985284480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319890.157468:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319890.157554:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985284480:12345-0@lo:38:kworker.0 Request processed in 110us (10332us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319890.157562:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5455 00000100:00100000:1.0:1767319890.157580:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5456 00000100:00100000:1.0:1767319890.157584:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985284864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319890.157593:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319890.157616:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985284864:12345-0@lo:38:kworker.0 Request processed in 31us (10192us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319890.157621:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5456 00000100:00080000:0.0:1767319890.158834:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.158843:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319890.158848:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985284480:0@lo:38:kworker.0 00000100:00080000:0.0:1767319890.158860:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.158863:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319890.158866:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985284608:0@lo:38:kworker.0 00000100:00080000:0.0:1767319890.158875:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.158878:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319890.158880:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985284864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319890.158974:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985285120 00000100:00100000:3.0:1767319890.158981:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985285120 00000100:00100000:3.0:1767319890.159019:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5457 00000100:00100000:3.0:1767319890.159025:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985285120:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319890.159040:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319890.159090:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985285120:12345-0@lo:38:kworker.0 Request processed in 65us (5804us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319890.159097:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5457 00000100:00080000:0.0:1767319890.159276:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.159281:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319890.159284:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985285120:0@lo:38:kworker.0 00010000:00010000:0.0:1767319890.779431:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767319891.114129:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920585984 offset 0 length 520: 4 00000100:00100000:2.0:1767319891.116003:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319891.116198:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319891.116200:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920586112 00000100:00100000:0.0:1767319891.116212:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920586112 00000100:00100000:0.0:1767319891.116876:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920586240 00000100:00100000:0.0:1767319891.116883:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920586240 00000100:00100000:1.0:1767319891.116893:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5458 00000100:00100000:1.0:1767319891.116903:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920586112:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319891.116937:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319891.117010:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920586112:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 106us (1015us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319891.117017:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5458 00000100:00100000:0.0:1767319891.117205:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4891 00000100:00100000:0.0:1767319891.117213:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920586240:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319891.117266:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920586240:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 54us (1067us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319891.117274:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4891 00000100:00080000:2.0:1767319891.117843:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319891.803745:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319892.828940:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319893.723570:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319893.723585:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985285504:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319893.723650:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319893.723685:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319893.723698:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985285632:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319893.723729:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319893.723793:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985285504 00000100:00100000:0.0:1767319893.723802:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985285504 00000100:00080000:2.0:1767319893.723854:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319893.723865:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985285632 00000100:00100000:0.0:1767319893.723870:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985285632 00000100:00080000:2.0:1767319893.723892:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319893.723906:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2382 00000100:00100000:0.0:1767319893.723920:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985285504:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319893.723922:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2383 00000100:00100000:2.0:1767319893.723938:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985285632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319893.723955:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:2.0:1767319893.723959:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319893.724010:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985285504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319893.724017:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2382 00000100:00100000:2.0:1767319893.724071:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985285632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 136us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319893.724074:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985285504:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319893.724082:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2383 00000100:00100000:1.0:1767319893.724149:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985285632:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319893.853593:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319894.875386:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319895.259420:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319895.259432:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319895.259435:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319895.259439:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319895.259443:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319895.259452:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5138 00000100:00080000:1.0:1767319895.259457:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319895.259463:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319895.259505:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae680 x1853163985285760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319895.259530:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319895.259534:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319895.259535:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319895.259540:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5138 00000100:00080000:1.0:1767319895.259543:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319895.259546:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319895.259558:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac000 x1853163985285888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319895.259606:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319895.259610:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319895.259611:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319895.259615:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5138 00000100:00080000:1.0:1767319895.259617:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319895.259620:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319895.259631:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985286144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319895.259661:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767319895.259663:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985286016:0@lo:400:kworker.0 00000100:00080000:1.0:1767319895.259668:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319895.259672:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319895.259676:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5138 00000100:00080000:1.0:1767319895.259679:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319895.259682:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319895.259694:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985286400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319895.259734:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985286528:0@lo:400:kworker.0 00000100:00100000:3.0:1767319895.259754:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319895.259773:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319895.259819:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319895.259826:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985286656:0@lo:400:kworker.0 00000100:00100000:3.0:1767319895.259851:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319895.259868:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286016 00000100:00100000:2.0:1767319895.259876:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286016 00000100:00100000:3.0:1767319895.259882:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286528 00000100:00100000:3.0:1767319895.259887:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286528 00000100:00100000:3.0:1767319895.259943:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286656 00000100:00100000:3.0:1767319895.259947:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286656 00000100:00100000:2.0:1767319895.259958:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4892 00000100:00100000:3.0:1767319895.259974:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3201 00000100:00100000:2.0:1767319895.259975:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985286016:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319895.259986:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985286528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319895.260038:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985286016:12345-0@lo:400:kworker.0 Request processed in 71us (284us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319895.260045:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985286528:12345-0@lo:400:kworker.0 Request processed in 59us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319895.260051:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4892 00000100:00100000:3.0:1767319895.260057:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3201 00000100:00100000:3.0:1767319895.260076:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3202 00000100:00080000:2.0:1767319895.260080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319895.260086:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985286656:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319895.260087:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319895.260093:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319895.260117:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985286656:12345-0@lo:400:kworker.0 Request processed in 34us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319895.260132:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3202 00000100:00100000:2.0:1767319895.260186:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985286656:0@lo:400:kworker.0 00000100:00100000:2.0:1767319895.260199:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985286016:0@lo:400:kworker.0 00000100:00100000:2.0:1767319895.260232:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985286272:0@lo:400:kworker.0 00000100:00100000:2.0:1767319895.260257:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319895.260362:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286272 00000100:00100000:3.0:1767319895.260367:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286272 00000100:00100000:3.0:1767319895.260405:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4893 00000100:00100000:3.0:1767319895.260412:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985286272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319895.260453:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985286272:12345-0@lo:400:kworker.0 Request processed in 41us (195us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319895.260463:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4893 00000100:00080000:2.0:1767319895.261577:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319895.261618:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985286272:0@lo:400:kworker.0 00000100:00100000:1.0:1767319895.262727:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985286528:0@lo:400:kworker.0 00000100:00100000:2.0:1767319895.263595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985285760:0@lo:38:kworker.0 00000100:00100000:2.0:1767319895.263657:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319895.263824:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985285888:0@lo:38:kworker.0 00000100:00100000:2.0:1767319895.263847:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319895.263875:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985286144:0@lo:38:kworker.0 00000100:00100000:2.0:1767319895.263893:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319895.263989:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985286400:0@lo:38:kworker.0 00000100:00100000:2.0:1767319895.264007:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319895.264056:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985285760 00000100:00100000:0.0:1767319895.264063:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985285760 00000100:00100000:0.0:1767319895.264092:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286144 00000100:00100000:0.0:1767319895.264094:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286144 00000100:00100000:0.0:1767319895.264109:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5459 00000100:00100000:0.0:1767319895.264115:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985285760:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319895.264136:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319895.264194:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985285760:12345-0@lo:38:kworker.0 Request processed in 77us (535us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319895.264195:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985285888 00000100:00100000:3.0:1767319895.264202:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985285888 00000100:00100000:0.0:1767319895.264204:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5459 00000100:00100000:0.0:1767319895.264220:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5460 00000100:00100000:0.0:1767319895.264225:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985286144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319895.264234:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319895.264243:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1855 00000100:00100000:3.0:1767319895.264250:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985285888:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319895.264256:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985286144:12345-0@lo:38:kworker.0 Request processed in 28us (360us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319895.264276:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5460 00010000:00020000:3.0:1767319895.264276:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319895.264314:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985285888:12345-0@lo:38:kworker.0 Request processed in 63us (464us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319895.264321:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1855 00000100:00080000:2.0:1767319895.264535:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319895.264541:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319895.264544:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985285760:0@lo:38:kworker.0 00000100:00080000:2.0:1767319895.264551:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319895.264554:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319895.264557:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985285888:0@lo:38:kworker.0 00000100:00080000:2.0:1767319895.264563:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319895.264566:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319895.264569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985286144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319895.264628:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286400 00000100:00100000:0.0:1767319895.264631:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286400 00000100:00100000:0.0:1767319895.264655:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5461 00000100:00100000:0.0:1767319895.264659:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985286400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319895.264669:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319895.264820:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985286400:12345-0@lo:38:kworker.0 Request processed in 159us (811us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319895.264826:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5461 00000100:00080000:2.0:1767319895.265001:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319895.265006:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319895.265009:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985286400:0@lo:38:kworker.0 00010000:00010000:0.0:1767319895.899544:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319896.234804:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319896.234859:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319896.234947:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920586368 00000100:00100000:1.0:1767319896.234957:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920586368 00000100:00100000:1.0:1767319896.234998:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920586496 00000100:00100000:1.0:1767319896.235001:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920586496 00000100:00100000:1.0:1767319896.235047:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4894 00000100:00100000:1.0:1767319896.235056:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920586496:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319896.235128:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920586496:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 73us (268us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319896.235135:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4894 00000100:00100000:1.0:1767319896.235153:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5462 00000100:00100000:1.0:1767319896.235157:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920586368:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319896.235205:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319896.235230:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920586368:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 72us (438us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319896.235237:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5462 00000100:00080000:2.0:1767319896.235319:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319896.924056:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319897.950012:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319898.843676:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985286784:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319898.843748:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319898.844054:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286784 00000100:00100000:3.0:1767319898.844063:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286784 00000100:00100000:3.0:1767319898.844136:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2384 00000100:00100000:3.0:1767319898.844146:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985286784:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319898.844215:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319898.844263:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985286784:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (514us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319898.844271:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2384 00000100:00080000:2.0:1767319898.844807:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319898.844849:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985286784:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319898.845128:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319898.845140:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985286912:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319898.845231:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319898.845521:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985286912 00000100:00100000:1.0:1767319898.845534:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985286912 00000100:00100000:1.0:1767319898.845588:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2385 00000100:00100000:1.0:1767319898.845597:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985286912:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319898.845617:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319898.845888:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985286912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 292us (655us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319898.845896:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2385 00000100:00100000:0.0:1767319898.845974:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985286912:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319898.846039:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319898.971832:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319899.995447:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319900.379393:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319900.379403:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319900.379407:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319900.379411:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319900.379415:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319900.379424:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5143 00000100:00080000:3.0:1767319900.379429:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319900.379435:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319900.379479:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffce00 x1853163985287040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319900.379520:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319900.379523:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319900.379525:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319900.379528:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5143 00000100:00080000:3.0:1767319900.379531:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319900.379534:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319900.379548:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985287168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319900.379587:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319900.379592:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319900.379599:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319900.379603:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5143 00000100:00080000:3.0:1767319900.379605:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319900.379609:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319900.379620:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffe300 x1853163985287424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319900.379646:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319900.379649:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319900.379651:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319900.379654:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5143 00000100:00080000:3.0:1767319900.379657:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319900.379660:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319900.379670:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff100 x1853163985287680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319900.379713:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985287808:0@lo:400:kworker.0 00000100:00100000:3.0:1767319900.379776:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319900.379834:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319900.379840:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985287552:0@lo:400:kworker.0 00000100:00100000:3.0:1767319900.379865:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319900.379903:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287808 00000100:00100000:3.0:1767319900.379910:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287808 00000100:00100000:3.0:1767319900.379962:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3203 00000100:00100000:3.0:1767319900.379970:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985287808:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319900.380019:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985287808:12345-0@lo:400:kworker.0 Request processed in 50us (243us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319900.380348:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3203 00000100:00100000:3.0:1767319900.380375:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287552 00000100:00100000:3.0:1767319900.380380:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287552 00000100:00100000:3.0:1767319900.380407:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4895 00000100:00100000:3.0:1767319900.380414:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985287552:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319900.380446:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985287552:12345-0@lo:400:kworker.0 Request processed in 33us (580us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319900.380452:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4895 00000100:00080000:3.0:1767319900.380468:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319900.380472:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319900.380591:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985287936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319900.380629:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319900.380653:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319900.380659:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985287296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319900.380680:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319900.380709:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287296 00000100:00100000:1.0:1767319900.380712:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287296 00000100:00100000:1.0:1767319900.380744:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4896 00000100:00100000:1.0:1767319900.380749:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985287296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319900.380781:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985287296:12345-0@lo:400:kworker.0 Request processed in 32us (100us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319900.380786:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4896 00000100:00100000:1.0:1767319900.380800:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287936 00000100:00100000:1.0:1767319900.380803:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287936 00000100:00100000:1.0:1767319900.380823:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3204 00000100:00100000:1.0:1767319900.380829:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985287936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319900.380855:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985287936:12345-0@lo:400:kworker.0 Request processed in 27us (226us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319900.380860:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3204 00000100:00080000:1.0:1767319900.380872:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319900.380875:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319900.380900:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985287296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319900.380910:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985287936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319900.381250:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985287040:0@lo:38:kworker.0 00000100:00100000:2.0:1767319900.381282:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319900.381303:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985287168:0@lo:38:kworker.0 00000100:00100000:2.0:1767319900.381324:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319900.381347:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985287424:0@lo:38:kworker.0 00000100:00100000:2.0:1767319900.381365:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319900.381379:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985287680:0@lo:38:kworker.0 00000100:00100000:2.0:1767319900.381395:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319900.381422:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287168 00000100:00100000:2.0:1767319900.381426:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287168 00000100:00100000:2.0:1767319900.381445:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1856 00000100:00100000:2.0:1767319900.381449:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985287168:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319900.381467:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319900.381523:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985287168:12345-0@lo:38:kworker.0 Request processed in 71us (197us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319900.381531:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1856 00000100:00100000:2.0:1767319900.381548:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287040 00000100:00100000:2.0:1767319900.381551:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287040 00000100:00100000:2.0:1767319900.381566:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287424 00000100:00100000:2.0:1767319900.381568:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287424 00000100:00100000:2.0:1767319900.381581:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985287680 00000100:00100000:2.0:1767319900.381583:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985287680 00000100:00100000:2.0:1767319900.381592:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5463 00000100:00100000:2.0:1767319900.381736:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985287040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319900.381748:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319900.381776:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985287040:12345-0@lo:38:kworker.0 Request processed in 40us (493us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319900.381781:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5463 00000100:00100000:2.0:1767319900.381794:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5464 00000100:00100000:2.0:1767319900.381797:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985287424:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319900.381805:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319900.381827:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985287424:12345-0@lo:38:kworker.0 Request processed in 28us (461us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319900.381832:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5464 00000100:00100000:2.0:1767319900.381843:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5465 00000100:00100000:2.0:1767319900.381847:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985287680:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319900.381855:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319900.381876:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985287680:12345-0@lo:38:kworker.0 Request processed in 28us (480us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319900.381882:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5465 00000100:00080000:2.0:1767319900.381913:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319900.381918:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319900.381921:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985287040:0@lo:38:kworker.0 00000100:00080000:2.0:1767319900.381929:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319900.381932:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319900.381935:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985287168:0@lo:38:kworker.0 00000100:00080000:2.0:1767319900.381941:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319900.381943:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319900.381945:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985287424:0@lo:38:kworker.0 00000100:00080000:2.0:1767319900.381952:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319900.381954:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319900.381957:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985287680:0@lo:38:kworker.0 00000100:00100000:3.0:1767319900.382252:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985287552:0@lo:400:kworker.0 00000100:00100000:3.0:1767319900.382264:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985287808:0@lo:400:kworker.0 00010000:00010000:2.0:1767319901.019663:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319901.354907:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319901.355020:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319901.355113:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920586752 00000100:00100000:3.0:1767319901.355129:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920586752 00000100:00100000:3.0:1767319901.355304:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920586624 00000100:00100000:3.0:1767319901.355309:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920586624 00000100:00100000:3.0:1767319901.355332:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4897 00000100:00100000:3.0:1767319901.355342:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920586752:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319901.355422:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920586752:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (538us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319901.355430:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4897 00000100:00100000:3.0:1767319901.355452:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5466 00000100:00100000:3.0:1767319901.355455:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920586624:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319901.355474:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319901.355497:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920586624:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (479us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319901.355504:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5466 00000100:00080000:1.0:1767319901.355880:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319902.043465:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319903.067389:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319903.963558:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319903.963587:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985288064:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319903.963663:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319903.963855:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288064 00000100:00100000:3.0:1767319903.963864:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288064 00000100:00100000:3.0:1767319903.963935:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2386 00000100:00100000:3.0:1767319903.963945:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985288064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319903.963975:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319903.964018:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985288064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (354us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319903.964025:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2386 00000100:00080000:1.0:1767319903.964134:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319903.964243:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985288064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319903.964712:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319903.964731:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985288192:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319903.964765:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319903.964932:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288192 00000100:00100000:1.0:1767319903.964938:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288192 00000100:00100000:1.0:1767319903.964975:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2387 00000100:00100000:1.0:1767319903.964983:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985288192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319903.965005:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319903.965041:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985288192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319903.965048:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2387 00000100:00080000:1.0:1767319903.965064:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319903.965142:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985288192:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319904.092089:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319905.115631:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319905.499610:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319905.499623:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319905.499627:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319905.499632:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319905.499637:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319905.499648:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5148 00000100:00080000:1.0:1767319905.499654:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319905.499662:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319905.499714:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13800 x1853163985288320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319905.499743:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319905.499747:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319905.499749:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319905.499752:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5148 00000100:00080000:1.0:1767319905.499756:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319905.499759:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319905.499770:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10700 x1853163985288448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319905.499815:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319905.499819:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319905.499821:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319905.499824:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5148 00000100:00080000:1.0:1767319905.499827:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319905.499830:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319905.499840:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13480 x1853163985288704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319905.499967:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985288320:0@lo:38:kworker.0 00000100:00100000:0.0:1767319905.500054:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319905.500140:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985288448:0@lo:38:kworker.0 00000100:00100000:0.0:1767319905.500191:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319905.500238:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288320 00000100:00100000:0.0:1767319905.500249:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288320 00000100:00100000:0.0:1767319905.500294:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5467 00000100:00100000:0.0:1767319905.500302:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985288320:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319905.500327:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319905.500367:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985288320:12345-0@lo:38:kworker.0 Request processed in 64us (312us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319905.500374:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5467 00000100:00100000:0.0:1767319905.500402:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288448 00000100:00100000:0.0:1767319905.500405:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288448 00000100:00100000:0.0:1767319905.500440:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1857 00000100:00100000:0.0:1767319905.500445:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985288448:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319905.500455:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319905.500479:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985288448:12345-0@lo:38:kworker.0 Request processed in 33us (288us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319905.500485:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1857 00000100:00080000:0.0:1767319905.500529:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319905.500533:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319905.500536:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985288320:0@lo:38:kworker.0 00000100:00080000:0.0:1767319905.500545:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319905.500548:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319905.500551:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985288448:0@lo:38:kworker.0 00000100:00080000:1.0:1767319905.500786:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319905.500791:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319905.500793:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319905.500796:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5148 00000100:00080000:1.0:1767319905.500799:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319905.500803:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319905.500818:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10000 x1853163985288960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319905.500946:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985288832:0@lo:400:kworker.0 00000100:00100000:1.0:1767319905.500980:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319905.501007:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319905.501011:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985289088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319905.501039:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319905.501082:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289088 00000100:00100000:1.0:1767319905.501087:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289088 00000100:00100000:1.0:1767319905.501138:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3205 00000100:00100000:1.0:1767319905.501145:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985289088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319905.501214:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985289088:12345-0@lo:400:kworker.0 Request processed in 69us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319905.501222:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3205 00000100:00100000:1.0:1767319905.501246:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288832 00000100:00100000:1.0:1767319905.501250:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288832 00000100:00100000:1.0:1767319905.501278:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4898 00000100:00100000:1.0:1767319905.501284:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985288832:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319905.501312:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985288832:12345-0@lo:400:kworker.0 Request processed in 28us (331us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319905.501318:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4898 00000100:00080000:1.0:1767319905.501332:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319905.501336:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319905.501354:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985289088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319905.501362:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985288832:0@lo:400:kworker.0 00000100:00100000:3.0:1767319905.501543:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985288576:0@lo:400:kworker.0 00000100:00100000:3.0:1767319905.501574:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319905.501595:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319905.501600:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985289216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319905.501618:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319905.501638:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289216 00000100:00100000:3.0:1767319905.501641:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289216 00000100:00100000:3.0:1767319905.501669:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3206 00000100:00100000:3.0:1767319905.501674:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985289216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319905.501703:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985289216:12345-0@lo:400:kworker.0 Request processed in 29us (84us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319905.501709:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3206 00000100:00080000:3.0:1767319905.501721:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319905.501729:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288576 00000100:00100000:3.0:1767319905.501732:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288576 00000100:00100000:3.0:1767319905.501753:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4899 00000100:00100000:3.0:1767319905.501758:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985288576:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319905.501780:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985288576:12345-0@lo:400:kworker.0 Request processed in 22us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319905.501786:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4899 00000100:00080000:3.0:1767319905.501796:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319905.501812:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985289216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319905.501819:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985288576:0@lo:400:kworker.0 00000100:00100000:0.0:1767319905.502212:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985288704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319905.502247:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319905.502274:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985288960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319905.502294:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319905.502391:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288704 00000100:00100000:3.0:1767319905.502396:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288704 00000100:00100000:3.0:1767319905.502516:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985288960 00000100:00100000:3.0:1767319905.502520:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985288960 00000100:00100000:3.0:1767319905.502544:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5468 00000100:00100000:3.0:1767319905.502550:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985288704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319905.502570:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319905.502609:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985288704:12345-0@lo:38:kworker.0 Request processed in 58us (360us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319905.502616:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5468 00000100:00100000:3.0:1767319905.502627:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5469 00000100:00100000:3.0:1767319905.502631:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985288960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319905.502639:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319905.502656:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985288960:12345-0@lo:38:kworker.0 Request processed in 24us (361us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319905.502662:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5469 00000100:00080000:0.0:1767319905.502767:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319905.502772:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319905.502776:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985288704:0@lo:38:kworker.0 00000100:00080000:0.0:1767319905.502783:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319905.502786:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319905.502789:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985288960:0@lo:38:kworker.0 00010000:00010000:1.0:1767319906.141224:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319906.475672:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319906.475795:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920587008 00000100:00100000:3.0:1767319906.475809:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920587008 00000100:00100000:3.0:1767319906.475910:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4900 00000100:00100000:3.0:1767319906.475921:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920587008:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319906.476033:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920587008:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (372us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319906.476042:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4900 00000100:00080000:0.0:1767319906.476221:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319906.478312:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319906.478433:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920586880 00000100:00100000:1.0:1767319906.478445:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920586880 00000100:00100000:1.0:1767319906.478503:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5470 00000100:00100000:1.0:1767319906.478513:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920586880:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319906.478541:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319906.478629:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920586880:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 116us (330us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319906.478638:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5470 00010000:00010000:2.0:1767319907.163938:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319908.187750:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319909.084301:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319909.084335:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985289344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319909.084432:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319909.084588:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289344 00000100:00100000:3.0:1767319909.084599:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289344 00000100:00100000:3.0:1767319909.084685:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2388 00000100:00100000:3.0:1767319909.084694:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985289344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319909.084731:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319909.084785:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985289344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (353us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319909.084794:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2388 00000100:00080000:2.0:1767319909.084859:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319909.084905:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985289344:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319909.085145:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319909.085157:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985289472:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319909.085230:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319909.085395:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289472 00000100:00100000:1.0:1767319909.085400:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289472 00000100:00100000:1.0:1767319909.085443:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2389 00000100:00100000:1.0:1767319909.085449:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9db480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985289472:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319909.085469:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319909.085508:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9db480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985289472:12345-0@lo:13:osp-pre-0-1.0 Request processed in 59us (278us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319909.085516:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2389 00000100:00100000:1.0:1767319909.085550:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985289472:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319909.085650:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767319909.211640:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319910.242584:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319910.619541:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319910.619557:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319910.619561:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319910.619566:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319910.619572:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319910.619584:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5153 00000100:00080000:2.0:1767319910.619590:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319910.619599:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319910.619659:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985289600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319910.619687:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319910.619690:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319910.619692:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319910.619696:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5153 00000100:00080000:2.0:1767319910.619698:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319910.619701:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319910.619714:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985289728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319910.619759:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319910.619763:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319910.619765:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319910.619769:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5153 00000100:00080000:2.0:1767319910.619771:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319910.619775:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319910.619796:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae680 x1853163985289984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319910.619832:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319910.619836:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319910.619838:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319910.619841:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5153 00000100:00080000:2.0:1767319910.619844:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319910.619847:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319910.619857:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad180 x1853163985290240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319910.619902:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985290112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319910.620000:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319910.620007:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985289856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319910.620051:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319910.620073:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319910.620091:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319910.620094:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985290368:0@lo:400:kworker.0 00000100:00100000:1.0:1767319910.620102:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985290496:0@lo:400:kworker.0 00000100:00100000:2.0:1767319910.620133:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319910.620154:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985290112 00000100:00100000:1.0:1767319910.620188:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985290112 00000100:00080000:2.0:1767319910.620224:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319910.620256:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289856 00000100:00100000:1.0:1767319910.620260:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289856 00000100:00100000:2.0:1767319910.620268:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4901 00000100:00100000:2.0:1767319910.620284:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985290112:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319910.620296:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4902 00000100:00100000:1.0:1767319910.620305:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985289856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319910.620379:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985289856:12345-0@lo:400:kworker.0 Request processed in 73us (326us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319910.620386:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985290112:12345-0@lo:400:kworker.0 Request processed in 105us (384us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319910.620466:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985290368 00000100:00100000:3.0:1767319910.620471:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985290368 00000100:00100000:3.0:1767319910.620505:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3207 00000100:00100000:3.0:1767319910.620511:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985290368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319910.620545:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985290368:12345-0@lo:400:kworker.0 Request processed in 35us (414us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319910.620550:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3207 00000100:00100000:3.0:1767319910.620624:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985290368:0@lo:400:kworker.0 00000100:00100000:3.0:1767319910.620636:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985290112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319910.620676:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319910.620701:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985289856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319910.620712:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4902 00000100:00100000:3.0:1767319910.620719:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985290496 00000100:00100000:3.0:1767319910.620724:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985290496 00000100:00080000:1.0:1767319910.620740:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319910.620753:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3208 00000100:00100000:3.0:1767319910.620762:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985290496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319910.620805:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985290496:12345-0@lo:400:kworker.0 Request processed in 45us (128us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319910.620812:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3208 00000100:00100000:1.0:1767319910.620842:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985290496:0@lo:400:kworker.0 00000100:00100000:0.0:1767319910.621590:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985289600:0@lo:38:kworker.0 00000100:00100000:0.0:1767319910.621639:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319910.621672:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985289728:0@lo:38:kworker.0 00000100:00100000:0.0:1767319910.621698:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319910.621733:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985289984:0@lo:38:kworker.0 00000100:00100000:0.0:1767319910.621945:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319910.622006:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985290240:0@lo:38:kworker.0 00000100:00100000:0.0:1767319910.622043:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319910.622280:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289600 00000100:00100000:1.0:1767319910.622292:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289600 00000100:00100000:3.0:1767319910.622383:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289728 00000100:00100000:3.0:1767319910.622390:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289728 00000100:00100000:0.0:1767319910.622556:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985289984 00000100:00100000:0.0:1767319910.622566:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985289984 00000100:00100000:3.0:1767319910.622632:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1858 00000100:00100000:3.0:1767319910.622642:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985289728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319910.622668:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319910.622731:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985289728:12345-0@lo:38:kworker.0 Request processed in 88us (1030us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319910.622738:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1858 00000100:00080000:3.0:1767319910.623360:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319910.623367:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319910.623371:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985289728:0@lo:38:kworker.0 00000100:00100000:2.0:1767319910.629679:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4901 00000100:00100000:2.0:1767319910.629736:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985290240 00000100:00100000:2.0:1767319910.629755:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985290240 00000100:00100000:2.0:1767319910.629821:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5471 00000100:00100000:2.0:1767319910.629831:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985290240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319910.629863:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319910.629975:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985290240:12345-0@lo:38:kworker.0 Request processed in 145us (7930us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319910.629990:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5471 00000100:00080000:3.0:1767319910.630299:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319910.630307:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319910.630313:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985290240:0@lo:38:kworker.0 00000100:00100000:1.0:1767319910.630437:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5472 00000100:00100000:1.0:1767319910.630446:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985289600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319910.630461:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319910.630524:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985289600:12345-0@lo:38:kworker.0 Request processed in 78us (8884us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319910.630532:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5472 00000100:00100000:0.0:1767319910.630924:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5473 00000100:00100000:0.0:1767319910.630939:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985289984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319910.630967:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319910.631051:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985289984:12345-0@lo:38:kworker.0 Request processed in 113us (9113us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319910.631058:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5473 00000100:00080000:3.0:1767319910.631956:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319910.631967:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319910.631972:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985289600:0@lo:38:kworker.0 00000100:00080000:3.0:1767319910.631999:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319910.632004:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319910.632007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985289984:0@lo:38:kworker.0 00010000:00010000:1.0:1767319911.259514:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319911.598652:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319911.598700:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319911.599041:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920587264 00000100:00100000:1.0:1767319911.599052:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920587264 00000100:00100000:1.0:1767319911.599133:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920587136 00000100:00100000:1.0:1767319911.599137:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920587136 00000100:00100000:1.0:1767319911.599194:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4903 00000100:00100000:1.0:1767319911.599206:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920587264:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319911.599283:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920587264:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (640us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319911.599290:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4903 00000100:00100000:1.0:1767319911.599325:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5474 00000100:00100000:1.0:1767319911.599331:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920587136:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319911.599350:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319911.599380:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920587136:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (680us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319911.599387:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5474 00000100:00080000:3.0:1767319911.599575:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319912.283366:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319913.307391:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319914.203442:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319914.203465:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985290624:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319914.203533:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319914.204138:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985290752:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319914.204216:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319914.205924:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985290752 00000100:00100000:1.0:1767319914.205925:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985290624 00000100:00100000:3.0:1767319914.205937:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985290752 00000100:00100000:1.0:1767319914.205938:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985290624 00000100:00100000:1.0:1767319914.206137:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2390 00000100:00100000:1.0:1767319914.206146:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985290624:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319914.206200:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319914.206266:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985290624:12345-0@lo:13:osp-pre-1-1.0 Request processed in 121us (2733us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319914.206274:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2390 00000100:00100000:1.0:1767319914.206326:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985290624:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319914.211420:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319914.211430:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319914.214456:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2391 00000100:00100000:3.0:1767319914.214475:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985290752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319914.214509:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319914.214598:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985290752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 129us (10381us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319914.214606:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2391 00000100:00100000:2.0:1767319914.214712:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985290752:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319914.331423:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319915.355504:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319915.694458:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767319915.694519:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920587392 offset 0 length 520: 4 00000100:00100000:3.0:1767319915.694544:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319915.699231:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920587648 00000100:00100000:0.0:1767319915.699248:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920587648 00000100:00100000:0.0:1767319915.699337:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920587520 00000100:00100000:0.0:1767319915.699341:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920587520 00000100:00100000:0.0:1767319915.699369:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4904 00000100:00100000:0.0:1767319915.699379:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9db100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920587648:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767319915.699476:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319915.699477:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9db100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920587648:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (5030us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319915.699488:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4904 00000100:00100000:0.0:1767319915.699508:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5475 00000100:00100000:0.0:1767319915.699513:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9da680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920587520:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319915.699534:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319915.699562:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9da680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920587520:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (5016us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319915.699569:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5475 00000100:00080000:0.0:1767319915.740338:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.740350:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319915.740354:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.740358:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.740363:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.740371:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5158 00000100:00080000:0.0:1767319915.740376:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.740384:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319915.740426:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985290880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319915.740443:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.740446:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.740448:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.740452:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5158 00000100:00080000:0.0:1767319915.740454:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.740458:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319915.740473:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4380 x1853163985291008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319915.740534:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.740538:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.740540:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.740544:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5158 00000100:00080000:0.0:1767319915.740546:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.740550:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319915.740562:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdad80 x1853163985291264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319915.740586:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.740589:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.740591:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.740594:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5158 00000100:00080000:0.0:1767319915.740596:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.740599:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319915.740608:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9f80 x1853163985291520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319915.741005:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985291776:0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.741038:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319890/real 1767319890] req@ffff8ce1a6a60380 x1853163985284352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319915 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319915.741066:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a6a60380 x1853163985284352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319915 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319915.741080:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.741087:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985290880:0@lo:38:kworker.0 00000100:00100000:3.0:1767319915.741123:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.741159:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319915.741159:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985291008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.741191:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985291136:0@lo:400:kworker.0 00000100:00100000:2.0:1767319915.741227:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.741233:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.741272:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985291264:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.741277:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985290880 00000100:00100000:2.0:1767319915.741286:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985290880 00000100:00100000:3.0:1767319915.741296:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.741315:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985291520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.741325:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985291136 00000100:00100000:2.0:1767319915.741329:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985291136 00000100:00100000:3.0:1767319915.741340:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319915.741359:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319915.741371:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319915.741373:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985291264 00000100:00100000:3.0:1767319915.741376:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985284352:0@lo:250:kworker.0 00000100:00100000:2.0:1767319915.741378:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985291264 00000100:00100000:2.0:1767319915.741390:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985291520 00000100:00100000:2.0:1767319915.741393:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985291520 00000100:00100000:2.0:1767319915.741406:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4905 00000100:00100000:2.0:1767319915.741416:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985291136:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.741423:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985291008 00000100:00100000:3.0:1767319915.741427:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985291008 00000100:00100000:3.0:1767319915.741452:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1859 00000100:00100000:3.0:1767319915.741459:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985291008:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.741474:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985291136:12345-0@lo:400:kworker.0 Request processed in 59us (246us total) trans 0 rc 0/0 00010000:00020000:3.0:1767319915.741478:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319915.741487:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4905 00000100:00100000:2.0:1767319915.741510:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5476 00000100:00100000:2.0:1767319915.741515:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985290880:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319915.741521:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985291008:12345-0@lo:38:kworker.0 Request processed in 60us (289us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767319915.741531:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319915.741531:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1859 00000100:00080000:3.0:1767319915.741570:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319915.741574:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319915.741577:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985291008:0@lo:38:kworker.0 00000100:00100000:0.0:1767319915.741617:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985291776 00000100:00100000:3.0:1767319915.741619:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5477 00000100:00100000:0.0:1767319915.741623:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985291776 00000100:00100000:3.0:1767319915.741627:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9da300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985291264:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319915.741638:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319915.741661:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3209 00000100:00100000:0.0:1767319915.741669:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985291776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319915.741718:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985291392:0@lo:400:kworker.0 00000100:00100000:2.0:1767319915.746507:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985290880:12345-0@lo:38:kworker.0 Request processed in 4979us (5372us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319915.746522:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5476 00000100:00100000:2.0:1767319915.746562:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5478 00000100:00100000:2.0:1767319915.746568:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9dad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985291520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319915.746589:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319915.746647:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9dad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985291520:12345-0@lo:38:kworker.0 Request processed in 77us (5307us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319915.746652:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5478 00000100:00100000:2.0:1767319915.746709:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985291136:0@lo:400:kworker.0 00000100:00080000:2.0:1767319915.746747:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.746752:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.746755:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985290880:0@lo:38:kworker.0 00000100:00080000:2.0:1767319915.746763:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.749446:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.749458:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985291520:0@lo:38:kworker.0 00000100:00080000:2.0:1767319915.749531:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319915.749578:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319915.749583:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319915.749587:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319915.749592:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5138 00000100:00080000:2.0:1767319915.749597:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319915.749602:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319915.749635:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ed80 x1853163985291904/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319915.749661:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319915.749664:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319915.749666:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319915.749670:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5163 00000100:00080000:2.0:1767319915.749672:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319915.749675:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319915.749690:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ea00 x1853163985292032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319915.749699:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319915.749702:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319915.749704:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319915.749707:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5163 00000100:00080000:2.0:1767319915.749710:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319915.749713:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319915.749725:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7df80 x1853163985292160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319915.749735:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319915.749738:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319915.749762:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319915.749765:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319915.749869:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319915.749873:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5163 00000100:00080000:2.0:1767319915.749876:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319915.749879:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319915.749900:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7f800 x1853163985292416/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319915.749941:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319915.749955:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985292288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319915.750016:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.750054:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292288 00000100:00100000:2.0:1767319915.750061:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292288 00000100:00100000:2.0:1767319915.750112:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4906 00000100:00100000:2.0:1767319915.750120:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985292288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319915.750188:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985292288:12345-0@lo:400:kworker.0 Request processed in 68us (171us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319915.750195:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4906 00000100:00100000:2.0:1767319915.750238:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985292288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319915.750256:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985291904:0@lo:250:kworker.0 00000400:00000100:2.0:1767319915.750277:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985291904 offset 0 length 520: 4 00000100:00100000:2.0:1767319915.750298:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985292032:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.750333:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.750349:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985292160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.750371:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.750395:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985292416:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.750415:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.751974:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292160 00000100:00100000:3.0:1767319915.751989:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292160 00000100:00100000:3.0:1767319915.752036:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1860 00000100:00100000:3.0:1767319915.752043:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985292160:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319915.752065:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319915.752126:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985292160:12345-0@lo:38:kworker.0 Request processed in 82us (1753us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319915.752133:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1860 00000100:00100000:0.0:1767319915.752208:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985291776:12345-0@lo:400:kworker.0 Request processed in 10537us (11127us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319915.752568:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3209 00000100:00100000:0.0:1767319915.752624:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985291776:0@lo:400:kworker.0 00000100:00100000:0.0:1767319915.752677:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985291648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319915.752715:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319915.752738:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985292672:0@lo:400:kworker.0 00000100:00100000:0.0:1767319915.752757:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319915.752777:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319915.752783:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985292544:0@lo:400:kworker.0 00000100:00100000:0.0:1767319915.752802:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319915.752932:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985291648 00000100:00100000:0.0:1767319915.752938:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985291648 00000100:00100000:0.0:1767319915.752968:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292672 00000100:00100000:0.0:1767319915.752971:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292672 00000100:00080000:2.0:1767319915.752976:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.752982:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.752985:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985292160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.753010:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292032 00000100:00100000:2.0:1767319915.753014:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292032 00000100:00100000:0.0:1767319915.753025:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292544 00000100:00100000:0.0:1767319915.753027:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292544 00000100:00100000:2.0:1767319915.753032:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292416 00000100:00100000:2.0:1767319915.753034:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292416 00000100:00100000:2.0:1767319915.753049:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5479 00000100:00100000:2.0:1767319915.753054:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985292032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319915.753070:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319915.753104:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985292032:12345-0@lo:38:kworker.0 Request processed in 49us (2771us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319915.753111:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5479 00000100:00100000:2.0:1767319915.753125:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5480 00000100:00100000:2.0:1767319915.753129:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985292416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319915.753138:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319915.753157:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3210 00000100:00100000:2.0:1767319915.753161:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985292416:12345-0@lo:38:kworker.0 Request processed in 32us (2746us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319915.753183:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5480 00000100:00100000:0.0:1767319915.753184:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+6:7014:x1853163985291648:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319915.753217:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.753222:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.753225:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985292032:0@lo:38:kworker.0 00000100:00080000:2.0:1767319915.753233:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.753235:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.753238:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985292416:0@lo:38:kworker.0 00000100:00080000:2.0:1767319915.753257:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319915.753287:0:70198:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3211 00000100:00100000:2.0:1767319915.753296:0:70198:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985292672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319915.753330:0:70198:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985292672:12345-0@lo:400:kworker.0 Request processed in 36us (571us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319915.753335:0:70198:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3211 00000100:00100000:2.0:1767319915.753348:0:70198:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3212 00000100:00100000:2.0:1767319915.753354:0:70198:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:lustre-MDT0001-mdtlov_UUID+6:7014:x1853163985292544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319915.753378:0:70198:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_014:lustre-MDT0001-mdtlov_UUID+6:7014:x1853163985292544:12345-0@lo:400:kworker.0 Request processed in 25us (575us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319915.753384:0:70198:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3212 00000100:00100000:0.0:1767319915.753433:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985291648:12345-0@lo:400:kworker.0 Request processed in 268us (716us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319915.753440:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3210 00000100:00100000:0.0:1767319915.753465:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985292544:0@lo:400:kworker.0 00000100:00100000:0.0:1767319915.753474:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985291648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319915.753482:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985292672:0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.755475:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9da300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985291264:12345-0@lo:38:kworker.0 Request processed in 13838us (14168us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319915.755490:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5477 00000100:00080000:2.0:1767319915.757430:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.757442:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.757447:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985291264:0@lo:38:kworker.0 00000100:00100000:1.0:1767319915.760245:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.760672:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985291392 00000100:00100000:3.0:1767319915.760681:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985291392 00000100:00100000:3.0:1767319915.760742:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4907 00000100:00100000:3.0:1767319915.760750:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985291392:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.760813:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985291392:12345-0@lo:400:kworker.0 Request processed in 64us (581us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319915.760820:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4907 00000100:00100000:1.0:1767319915.760867:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985291392:0@lo:400:kworker.0 00010000:00010000:1.0:1767319916.382175:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319917.403412:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319918.429315:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319919.323530:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985292928:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319919.323600:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319919.323649:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319919.323655:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985292800:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319919.323678:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319919.323751:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292928 00000100:00100000:3.0:1767319919.323759:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292928 00000100:00100000:3.0:1767319919.323844:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985292800 00000100:00100000:3.0:1767319919.323848:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985292800 00000100:00100000:3.0:1767319919.323871:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2392 00000100:00100000:3.0:1767319919.323879:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985292928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319919.323907:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319919.323949:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985292928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (348us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319919.323955:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2392 00000100:00100000:3.0:1767319919.323970:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2393 00000100:00100000:3.0:1767319919.323976:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985292800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319919.323988:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319919.324011:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985292800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 36us (333us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319919.324017:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2393 00000100:00080000:1.0:1767319919.324650:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319919.324655:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319919.324699:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985292800:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319919.324713:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985292928:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319919.452202:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319919.771697:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319919.771708:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319919.771711:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319919.771715:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319919.771720:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319919.771728:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5164 00000100:00080000:2.0:1767319919.771734:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319919.771740:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319919.771780:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7f800 x1853163985293056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319919.772677:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985293056:0@lo:38:kworker.0 00000100:00100000:0.0:1767319919.772749:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319919.772876:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293056 00000100:00100000:1.0:1767319919.772886:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293056 00000100:00100000:1.0:1767319919.772938:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5481 00000100:00100000:1.0:1767319919.772947:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985293056:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319919.772967:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319919.773016:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319919.773020:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319919.773023:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319919.773027:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5164 00000100:00080000:2.0:1767319919.773030:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319919.773036:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319919.773056:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ea00 x1853163985293184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319919.777112:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985293184:0@lo:38:kworker.0 00000100:00100000:0.0:1767319919.777220:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319919.777320:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293184 00000100:00100000:3.0:1767319919.777338:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293184 00000100:00100000:3.0:1767319919.777376:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1861 00000100:00100000:3.0:1767319919.777384:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985293184:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319919.777420:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319919.777463:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985293184:12345-0@lo:38:kworker.0 Request processed in 78us (243us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319919.777470:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1861 00000100:00100000:1.0:1767319919.778119:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985293056:12345-0@lo:38:kworker.0 Request processed in 5169us (5367us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319919.778129:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5481 00000100:00080000:2.0:1767319919.778279:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319919.778284:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319919.778287:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319919.778292:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5163 00000100:00080000:2.0:1767319919.778296:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319919.778301:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319919.778322:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7f480 x1853163985293440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319919.778352:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319919.778355:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319919.778357:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319919.778360:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5164 00000100:00080000:2.0:1767319919.778362:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319919.778365:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319919.778377:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7d880 x1853163985293696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319919.778594:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985293312:0@lo:400:kworker.0 00000100:00100000:1.0:1767319919.778640:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319919.778670:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293312 00000100:00100000:1.0:1767319919.778676:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293312 00000100:00100000:1.0:1767319919.778720:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4908 00000100:00100000:1.0:1767319919.778728:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985293312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319919.778770:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985293312:12345-0@lo:400:kworker.0 Request processed in 43us (129us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319919.778777:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4908 00000100:00080000:1.0:1767319919.778816:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319919.778847:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985293312:0@lo:400:kworker.0 00000100:00100000:1.0:1767319919.778877:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319919.778883:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985293568:0@lo:400:kworker.0 00000100:00100000:1.0:1767319919.778904:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319919.778922:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293568 00000100:00100000:1.0:1767319919.778925:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293568 00000100:00100000:1.0:1767319919.778949:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4909 00000100:00100000:1.0:1767319919.778955:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985293568:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319919.778982:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985293568:12345-0@lo:400:kworker.0 Request processed in 27us (76us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319919.778988:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4909 00000100:00080000:1.0:1767319919.779000:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319919.779017:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985293568:0@lo:400:kworker.0 00000100:00100000:2.0:1767319919.779347:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985293824:0@lo:400:kworker.0 00000100:00100000:2.0:1767319919.779382:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319919.779421:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319919.779427:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985293952:0@lo:400:kworker.0 00000100:00100000:2.0:1767319919.779447:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319919.779472:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293824 00000100:00100000:2.0:1767319919.779476:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293824 00000100:00100000:2.0:1767319919.779503:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293952 00000100:00100000:2.0:1767319919.779506:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293952 00000100:00100000:2.0:1767319919.779523:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3213 00000100:00100000:2.0:1767319919.779529:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985293824:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319919.779560:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985293824:12345-0@lo:400:kworker.0 Request processed in 32us (178us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319919.779566:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3213 00000100:00100000:2.0:1767319919.779578:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3214 00000100:00100000:2.0:1767319919.779583:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985293952:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319919.779606:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985293952:12345-0@lo:400:kworker.0 Request processed in 23us (158us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319919.779611:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3214 00000100:00100000:2.0:1767319919.779642:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985293952:0@lo:400:kworker.0 00000100:00100000:2.0:1767319919.779651:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985293824:0@lo:400:kworker.0 00000100:00080000:0.0:1767319919.789065:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319919.789077:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319919.789082:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985293056:0@lo:38:kworker.0 00000100:00100000:0.0:1767319919.789120:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985293440:0@lo:38:kworker.0 00000100:00100000:0.0:1767319919.789204:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319919.789289:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985293696:0@lo:38:kworker.0 00000100:00100000:0.0:1767319919.789312:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319919.802258:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319919.802271:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319919.802276:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985293184:0@lo:38:kworker.0 00000100:00100000:1.0:1767319919.812492:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293440 00000100:00100000:1.0:1767319919.812505:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293440 00000100:00100000:1.0:1767319919.812591:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985293696 00000100:00100000:1.0:1767319919.812595:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985293696 00000100:00100000:1.0:1767319919.812624:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5482 00000100:00100000:1.0:1767319919.812632:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985293440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319919.812657:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319919.812735:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985293440:12345-0@lo:38:kworker.0 Request processed in 103us (23530us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319919.812743:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5482 00000100:00100000:1.0:1767319919.812759:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5483 00000100:00100000:1.0:1767319919.812763:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985293696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319919.812772:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319919.812794:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985293696:12345-0@lo:38:kworker.0 Request processed in 30us (23482us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319919.812800:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5483 00000100:00080000:0.0:1767319919.813058:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319919.813064:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319919.813068:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985293440:0@lo:38:kworker.0 00000100:00080000:0.0:1767319919.813080:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319919.813083:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319919.813086:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985293696:0@lo:38:kworker.0 00010000:00010000:0.0:1767319920.475767:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319920.796020:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319920.796031:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319920.796036:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319920.796040:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319920.796044:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319920.796052:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5168 00000100:00080000:2.0:1767319920.796058:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319920.796065:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319920.796116:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7f100 x1853163985294080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319920.796207:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985294080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319920.796279:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.796436:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294080 00000100:00100000:3.0:1767319920.796448:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294080 00000100:00100000:3.0:1767319920.796502:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5484 00000100:00100000:3.0:1767319920.796511:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985294080:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319920.796631:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319920.797689:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319920.797699:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319920.797702:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319920.797707:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5168 00000100:00080000:2.0:1767319920.797711:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319920.797716:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319920.797747:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7e680 x1853163985294208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319920.800633:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985294208:0@lo:38:kworker.0 00000100:00100000:0.0:1767319920.800727:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319920.801293:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294208 00000100:00100000:1.0:1767319920.801302:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294208 00000100:00100000:1.0:1767319920.801358:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1862 00000100:00100000:1.0:1767319920.801366:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985294208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319920.801390:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319920.801439:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985294208:12345-0@lo:38:kworker.0 Request processed in 72us (711us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319920.801448:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1862 00000100:00100000:3.0:1767319920.804474:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985294080:12345-0@lo:38:kworker.0 Request processed in 7954us (8185us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319920.804488:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5484 00000100:00080000:0.0:1767319920.805118:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319920.805125:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319920.805129:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985294080:0@lo:38:kworker.0 00000100:00080000:0.0:1767319920.805161:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319920.805179:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319920.805182:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985294208:0@lo:38:kworker.0 00000100:00100000:1.0:1767319920.805415:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985294336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319920.805478:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.805574:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294336 00000100:00100000:3.0:1767319920.805580:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294336 00000100:00100000:3.0:1767319920.805627:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4910 00000100:00100000:3.0:1767319920.805636:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985294336:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319920.805813:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319920.805818:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319920.805821:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319920.805826:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5168 00000100:00080000:2.0:1767319920.805830:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319920.805834:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319920.805867:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7d500 x1853163985294464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319920.805933:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985294592:0@lo:400:kworker.0 00000100:00100000:0.0:1767319920.805962:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319920.805992:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294592 00000100:00100000:0.0:1767319920.805996:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294592 00000100:00100000:0.0:1767319920.806024:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4911 00000100:00100000:0.0:1767319920.806030:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985294592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319920.806064:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985294592:12345-0@lo:400:kworker.0 Request processed in 34us (101us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319920.806071:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4911 00000100:00100000:0.0:1767319920.806107:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985294592:0@lo:400:kworker.0 00000100:00100000:0.0:1767319920.806123:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985294464:0@lo:38:kworker.0 00000100:00100000:0.0:1767319920.806147:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319920.806186:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294464 00000100:00100000:0.0:1767319920.806190:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294464 00000100:00100000:0.0:1767319920.806209:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5485 00000100:00100000:0.0:1767319920.806214:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985294464:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319920.806234:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319920.806263:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985294464:12345-0@lo:38:kworker.0 Request processed in 48us (115us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319920.806270:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5485 00000100:00080000:0.0:1767319920.806294:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319920.806298:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319920.806301:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985294464:0@lo:38:kworker.0 00000100:00100000:3.0:1767319920.809806:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985294336:12345-0@lo:400:kworker.0 Request processed in 4152us (4308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319920.809820:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4910 00000100:00080000:2.0:1767319920.810869:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319920.810877:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319920.810881:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319920.810887:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5168 00000100:00080000:2.0:1767319920.810892:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319920.810898:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767319920.810941:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7f100 x1853163985294720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319920.811056:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985294976:0@lo:400:kworker.0 00000100:00100000:2.0:1767319920.811119:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319920.811154:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319920.811160:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985294848:0@lo:400:kworker.0 00000100:00100000:2.0:1767319920.811202:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319920.811230:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294976 00000100:00100000:2.0:1767319920.811237:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294976 00000100:00100000:2.0:1767319920.811278:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294848 00000100:00100000:2.0:1767319920.811282:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294848 00000100:00100000:2.0:1767319920.811301:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3215 00000100:00100000:2.0:1767319920.811309:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985294976:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319920.811359:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985294976:12345-0@lo:400:kworker.0 Request processed in 51us (240us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319920.811366:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3215 00000100:00100000:2.0:1767319920.811382:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3216 00000100:00100000:2.0:1767319920.811388:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985294848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319920.811413:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985294848:12345-0@lo:400:kworker.0 Request processed in 26us (211us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319920.811420:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3216 00000100:00100000:2.0:1767319920.811467:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985294848:0@lo:400:kworker.0 00000100:00100000:2.0:1767319920.811478:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985294976:0@lo:400:kworker.0 00000100:00100000:0.0:1767319920.811973:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985294720:0@lo:38:kworker.0 00000100:00100000:0.0:1767319920.812008:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.812082:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985294720 00000100:00100000:3.0:1767319920.812088:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985294720 00000100:00100000:3.0:1767319920.812119:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5486 00000100:00100000:3.0:1767319920.812124:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985294720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319920.812143:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319920.812211:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985294720:12345-0@lo:38:kworker.0 Request processed in 85us (201us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319920.812217:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5486 00000100:00080000:2.0:1767319920.812286:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319920.812292:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319920.812295:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985294720:0@lo:38:kworker.0 00000100:00100000:1.0:1767319920.816754:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985294336:0@lo:400:kworker.0 00000100:00100000:0.0:1767319920.820994:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319920.821047:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319920.821125:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920587904 00000100:00100000:3.0:1767319920.821134:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920587904 00000100:00100000:3.0:1767319920.821259:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920587776 00000100:00100000:3.0:1767319920.821262:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920587776 00000100:00100000:3.0:1767319920.821279:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4912 00000100:00100000:3.0:1767319920.821290:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920587904:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319920.821380:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920587904:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 92us (399us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319920.821389:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4912 00000100:00100000:3.0:1767319920.821408:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5487 00000100:00100000:3.0:1767319920.821412:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920587776:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319920.821432:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319920.821456:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920587776:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (408us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319920.821462:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5487 00000100:00080000:0.0:1767319920.821740:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319921.500581:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319922.523411:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319923.547462:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319924.443949:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985295104:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319924.444016:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319924.444098:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319924.444105:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985295232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319924.444127:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319924.444206:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985295104 00000100:00100000:0.0:1767319924.444214:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985295104 00000100:00100000:0.0:1767319924.444275:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985295232 00000100:00100000:0.0:1767319924.444278:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985295232 00000100:00100000:0.0:1767319924.444303:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2394 00000100:00100000:0.0:1767319924.444312:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985295104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319924.444348:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319924.444394:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985295104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (377us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319924.444401:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2394 00000100:00100000:0.0:1767319924.444419:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2395 00000100:00100000:0.0:1767319924.444425:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985295232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319924.444442:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319924.444479:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985295232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (351us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319924.444485:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2395 00000100:00080000:1.0:1767319924.444751:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319924.444756:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319924.444791:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985295232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319924.444805:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985295104:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319924.571538:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319925.596190:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319925.938289:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319925.938333:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319925.942071:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920588160 00000100:00100000:0.0:1767319925.942090:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920588160 00000100:00080000:2.0:1767319925.942300:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319925.942400:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920588032 00000100:00100000:0.0:1767319925.942404:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920588032 00000100:00100000:0.0:1767319925.942440:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4913 00000100:00100000:0.0:1767319925.942451:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920588160:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319925.942554:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920588160:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 104us (4274us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319925.942562:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4913 00000100:00100000:0.0:1767319925.942593:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5488 00000100:00100000:0.0:1767319925.942597:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9db800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920588032:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319925.942619:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319925.942648:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9db800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920588032:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (4313us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319925.942654:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5488 00000100:00080000:3.0:1767319925.979384:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319925.979399:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319925.979402:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319925.979407:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319925.979411:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319925.979419:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5169 00000100:00080000:3.0:1767319925.979424:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319925.979431:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319925.979472:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9500 x1853163985295360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319925.979501:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319925.979505:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319925.979506:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319925.979510:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5169 00000100:00080000:3.0:1767319925.979512:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319925.979515:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319925.979528:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9dbb80 x1853163985295488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319925.979776:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319925.979828:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985295360:0@lo:38:kworker.0 00000100:00100000:2.0:1767319925.979901:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319925.979960:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985295488:0@lo:38:kworker.0 00000100:00100000:2.0:1767319925.979984:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319925.980034:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985295488 00000100:00100000:2.0:1767319925.980042:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985295488 00000100:00100000:2.0:1767319925.980076:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1863 00000100:00100000:2.0:1767319925.980083:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985295488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319925.980103:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319925.980144:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985295488:12345-0@lo:38:kworker.0 Request processed in 59us (157us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319925.980151:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1863 00000100:00080000:2.0:1767319925.980221:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319925.980226:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319925.980229:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985295488:0@lo:38:kworker.0 00000100:00100000:2.0:1767319925.980266:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985295360 00000100:00100000:2.0:1767319925.980270:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985295360 00000100:00100000:2.0:1767319925.980291:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5489 00000100:00100000:2.0:1767319925.980295:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985295360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319925.980305:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319925.980334:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985295360:12345-0@lo:38:kworker.0 Request processed in 37us (433us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319925.980340:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5489 00000100:00080000:2.0:1767319925.980364:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319925.980367:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319925.980370:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985295360:0@lo:38:kworker.0 00000100:00080000:3.0:1767319925.980435:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319925.980439:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319925.980441:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319925.980445:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5169 00000100:00080000:3.0:1767319925.980447:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319925.980450:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319925.980468:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9500 x1853163985295744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319925.980529:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985295744:0@lo:38:kworker.0 00000100:00100000:2.0:1767319925.980562:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319925.980599:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985295744 00000100:00100000:2.0:1767319925.980603:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985295744 00000100:00100000:2.0:1767319925.980620:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5490 00000100:00100000:2.0:1767319925.980624:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985295744:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319925.980634:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319925.980662:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985295744:12345-0@lo:38:kworker.0 Request processed in 36us (99us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319925.980667:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5490 00000100:00080000:2.0:1767319925.980692:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319925.980706:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319925.980709:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985295744:0@lo:38:kworker.0 00000100:00100000:0.0:1767319925.980858:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319925.980868:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985295872:0@lo:400:kworker.0 00000100:00100000:0.0:1767319925.980900:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319925.980943:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985295872 00000100:00100000:0.0:1767319925.980947:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985295872 00000100:00100000:0.0:1767319925.980986:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4914 00000100:00100000:0.0:1767319925.980992:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985295872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319925.981031:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985295872:12345-0@lo:400:kworker.0 Request processed in 40us (131us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319925.981037:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4914 00000100:00080000:0.0:1767319925.981055:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319925.981076:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985295872:0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.985189:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985295616:0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.985280:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319925.985476:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985295616 00000100:00100000:2.0:1767319925.985486:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985295616 00000100:00100000:1.0:1767319925.985716:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4915 00000100:00100000:1.0:1767319925.985728:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985295616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.985785:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985295616:12345-0@lo:400:kworker.0 Request processed in 60us (504us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319925.985791:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4915 00000100:00100000:1.0:1767319925.985835:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985295616:0@lo:400:kworker.0 00000100:00080000:0.0:1767319925.985914:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319925.986037:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319925.986043:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319925.986047:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319925.986052:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5169 00000100:00080000:3.0:1767319925.986056:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319925.986061:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319925.986089:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9f80 x1853163985296000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319925.986139:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985296000:0@lo:38:kworker.0 00000100:00100000:2.0:1767319925.986189:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319925.986240:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296000 00000100:00100000:1.0:1767319925.986245:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296000 00000100:00100000:1.0:1767319925.986263:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5491 00000100:00100000:1.0:1767319925.986268:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985296000:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319925.986286:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319925.986378:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985296256:0@lo:400:kworker.0 00000100:00100000:3.0:1767319925.986409:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319925.986431:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319925.986437:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985296128:0@lo:400:kworker.0 00000100:00100000:3.0:1767319925.986458:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319925.986483:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296256 00000100:00100000:3.0:1767319925.986487:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296256 00000100:00100000:3.0:1767319925.986514:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296128 00000100:00100000:3.0:1767319925.986517:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296128 00000100:00100000:3.0:1767319925.986548:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3217 00000100:00100000:3.0:1767319925.986555:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985296256:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319925.986600:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985296256:12345-0@lo:400:kworker.0 Request processed in 45us (189us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319925.986607:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3217 00000100:00100000:3.0:1767319925.986620:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3218 00000100:00100000:3.0:1767319925.986626:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985296128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319925.986651:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985296128:12345-0@lo:400:kworker.0 Request processed in 25us (192us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319925.986656:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3218 00000100:00100000:3.0:1767319925.986705:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985296128:0@lo:400:kworker.0 00000100:00100000:3.0:1767319925.986716:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985296256:0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.998777:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985296000:12345-0@lo:38:kworker.0 Request processed in 12495us (12574us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319925.998795:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5491 00000100:00080000:2.0:1767319925.998951:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319925.998958:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319925.998963:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985296000:0@lo:38:kworker.0 00010000:00010000:0.0:1767319926.619398:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319927.643393:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319928.668548:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319929.564160:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319929.564203:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985296384:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319929.564274:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319929.565179:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319929.565193:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985296512:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319929.565233:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319929.571380:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296384 00000100:00100000:3.0:1767319929.571397:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296384 00000100:00100000:3.0:1767319929.571475:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296512 00000100:00100000:3.0:1767319929.571479:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296512 00000100:00100000:3.0:1767319929.571508:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2396 00000100:00100000:3.0:1767319929.571517:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985296384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319929.571560:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319929.571644:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985296384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 127us (7369us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319929.571651:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2396 00000100:00100000:3.0:1767319929.571669:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2397 00000100:00100000:3.0:1767319929.571676:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985296512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319929.571691:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319929.571725:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985296512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (6491us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319929.571731:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2397 00000100:00100000:3.0:1767319929.571780:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985296512:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319929.572737:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319929.572743:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319929.572786:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985296384:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319929.691444:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319930.716006:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319931.049881:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319931.049942:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319931.050438:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920588416 00000100:00100000:0.0:1767319931.050450:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920588416 00000100:00100000:0.0:1767319931.050528:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920588288 00000100:00100000:0.0:1767319931.050532:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920588288 00000100:00100000:0.0:1767319931.050565:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4916 00000100:00100000:0.0:1767319931.050576:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9daa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920588416:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319931.050668:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9daa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920588416:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 94us (798us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319931.050677:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4916 00000100:00100000:0.0:1767319931.050697:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5492 00000100:00100000:0.0:1767319931.050702:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9dbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920588288:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319931.050723:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319931.050747:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9dbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920588288:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (805us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319931.050754:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5492 00000100:00080000:2.0:1767319931.051329:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319931.099286:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319931.099297:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319931.099300:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319931.099304:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319931.099308:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319931.099317:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5174 00000100:00080000:3.0:1767319931.099320:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319931.099327:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319931.099365:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9500 x1853163985296640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319931.099395:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319931.099398:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319931.099399:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319931.099402:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5174 00000100:00080000:3.0:1767319931.099404:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319931.099407:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319931.099420:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9880 x1853163985296768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319931.099467:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319931.099471:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319931.099473:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319931.099477:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5174 00000100:00080000:3.0:1767319931.099480:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319931.099483:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319931.099496:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985297024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319931.099512:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985296896:0@lo:400:kworker.0 00000100:00080000:3.0:1767319931.099513:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319931.099519:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319931.099522:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319931.099527:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5174 00000100:00080000:3.0:1767319931.099529:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319931.099532:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319931.099545:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60380 x1853163985297280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319931.099587:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319931.099624:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985297408:0@lo:400:kworker.0 00000100:00100000:3.0:1767319931.099666:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319931.099674:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319931.099684:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985297152:0@lo:400:kworker.0 00000100:00100000:3.0:1767319931.099691:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319931.099699:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985297536:0@lo:400:kworker.0 00000100:00100000:1.0:1767319931.099713:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319931.099732:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319931.099755:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296896 00000100:00100000:1.0:1767319931.099763:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296896 00000100:00100000:3.0:1767319931.099770:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297408 00000100:00100000:3.0:1767319931.099776:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297408 00000100:00100000:1.0:1767319931.099820:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297152 00000100:00100000:3.0:1767319931.099822:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297536 00000100:00100000:1.0:1767319931.099825:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297152 00000100:00100000:3.0:1767319931.099827:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297536 00000100:00100000:3.0:1767319931.099868:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3219 00000100:00100000:1.0:1767319931.099869:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4917 00000100:00100000:3.0:1767319931.099878:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985297408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319931.099882:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985296896:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319931.099944:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985297408:12345-0@lo:400:kworker.0 Request processed in 68us (277us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319931.099951:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3219 00000100:00100000:3.0:1767319931.099967:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3220 00000100:00100000:3.0:1767319931.099973:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985297536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319931.099997:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985297536:12345-0@lo:400:kworker.0 Request processed in 25us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319931.100003:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3220 00000100:00100000:3.0:1767319931.100022:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4918 00000100:00100000:3.0:1767319931.100029:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985297152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319931.100055:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985297152:12345-0@lo:400:kworker.0 Request processed in 27us (343us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319931.100061:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4918 00000100:00100000:3.0:1767319931.100102:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985297536:0@lo:400:kworker.0 00000100:00100000:3.0:1767319931.100112:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985297408:0@lo:400:kworker.0 00000100:00100000:1.0:1767319931.100288:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985296896:12345-0@lo:400:kworker.0 Request processed in 61us (351us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319931.100298:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4917 00000100:00100000:1.0:1767319931.100328:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985297152:0@lo:400:kworker.0 00000100:00100000:1.0:1767319931.100337:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985296896:0@lo:400:kworker.0 00000100:00100000:0.0:1767319931.100506:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985296640:0@lo:38:kworker.0 00000100:00100000:0.0:1767319931.100540:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319931.100585:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985296768:0@lo:38:kworker.0 00000100:00100000:0.0:1767319931.100619:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319931.100662:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296640 00000100:00100000:3.0:1767319931.100668:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296640 00000100:00100000:3.0:1767319931.100689:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5493 00000100:00100000:3.0:1767319931.100693:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985296640:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319931.100715:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319931.100743:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985296640:12345-0@lo:38:kworker.0 Request processed in 49us (203us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319931.100749:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5493 00000100:00080000:2.0:1767319931.105197:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319931.105209:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319931.105213:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319931.106743:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985297024:0@lo:38:kworker.0 00000100:00100000:0.0:1767319931.106805:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319931.106842:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985297280:0@lo:38:kworker.0 00000100:00100000:0.0:1767319931.106887:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319931.106937:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319931.106942:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319931.106947:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985296640:0@lo:38:kworker.0 00000100:00100000:0.0:1767319931.107000:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985296768 00000100:00100000:0.0:1767319931.107007:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985296768 00000100:00100000:0.0:1767319931.107043:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1864 00000100:00100000:0.0:1767319931.107050:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985296768:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319931.107056:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297024 00000100:00100000:1.0:1767319931.107062:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297024 00010000:00020000:0.0:1767319931.107072:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319931.107100:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297280 00000100:00100000:1.0:1767319931.107104:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297280 00000100:00100000:1.0:1767319931.107121:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5494 00000100:00100000:1.0:1767319931.107127:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985297024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319931.107138:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319931.107279:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985296768:12345-0@lo:38:kworker.0 Request processed in 66us (6496us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319931.107289:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1864 00000100:00080000:0.0:1767319931.107320:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319931.107325:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319931.107328:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985296768:0@lo:38:kworker.0 00000100:00080000:0.0:1767319931.107337:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319931.107340:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319931.107343:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985297024:0@lo:38:kworker.0 00000100:00100000:1.0:1767319931.107524:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985297024:12345-0@lo:38:kworker.0 Request processed in 395us (718us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319931.107531:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5494 00000100:00100000:0.0:1767319931.108124:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5495 00000100:00100000:0.0:1767319931.108130:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985297280:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319931.108143:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319931.108229:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985297280:12345-0@lo:38:kworker.0 Request processed in 96us (1340us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319931.108236:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5495 00000100:00080000:1.0:1767319931.108378:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319931.108384:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319931.108388:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985297280:0@lo:38:kworker.0 00010000:00010000:2.0:1767319931.739722:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319932.763809:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319933.611103:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319933.611192:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920588544 00000100:00100000:3.0:1767319933.611202:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920588544 00000100:00100000:3.0:1767319933.611274:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5496 00000100:00100000:3.0:1767319933.611284:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920588544:12345-192.168.204.26@tcp:17:kworker.0 00000100:00080000:2.0:1767319933.611480:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319933.611513:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920588544:12345-192.168.204.26@tcp:17:kworker.0 Request processed in 231us (419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319933.611520:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5496 00010000:00010000:0.0:1767319933.787835:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319934.684572:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985297664:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319934.684648:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319934.684919:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297664 00000100:00100000:3.0:1767319934.684937:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297664 00000100:00100000:3.0:1767319934.684995:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2398 00000100:00100000:3.0:1767319934.685003:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985297664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319934.685032:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319934.685076:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985297664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (427us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319934.685084:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2398 00000100:00080000:2.0:1767319934.685143:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319934.685199:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985297664:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319934.685865:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985297792:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319934.685932:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319934.686003:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297792 00000100:00100000:3.0:1767319934.686010:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297792 00000100:00100000:3.0:1767319934.686068:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2399 00000100:00100000:3.0:1767319934.686076:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985297792:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319934.686099:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319934.686131:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985297792:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (199us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319934.686138:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2399 00000100:00080000:0.0:1767319934.687333:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319934.690192:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985297792:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319934.812746:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319935.835389:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319936.172336:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319936.172385:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319936.172564:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920588800 00000100:00100000:3.0:1767319936.172574:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920588800 00000100:00100000:3.0:1767319936.172693:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920588672 00000100:00100000:3.0:1767319936.172697:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920588672 00000100:00080000:0.0:1767319936.172737:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319936.173363:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4919 00000100:00100000:1.0:1767319936.173480:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5497 00000100:00100000:1.0:1767319936.173488:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920588672:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319936.173522:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319936.173599:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920588672:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 109us (1212us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319936.173607:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5497 00000100:00100000:0.0:1767319936.178748:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920588800:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319936.179027:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920588800:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 5651us (6698us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319936.179042:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4919 00000100:00080000:1.0:1767319936.219530:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319936.219542:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319936.219545:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319936.219549:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319936.219554:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319936.219563:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5179 00000100:00080000:1.0:1767319936.219568:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319936.219576:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319936.219940:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985297920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319936.219966:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319936.219970:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319936.219972:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319936.219975:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5179 00000100:00080000:1.0:1767319936.219978:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319936.219982:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319936.219995:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffce00 x1853163985298048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319936.220036:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319936.220040:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319936.220042:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319936.220046:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5179 00000100:00080000:1.0:1767319936.220048:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319936.220051:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319936.220063:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff480 x1853163985298304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319936.220083:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319936.220087:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319936.220089:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319936.220092:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5179 00000100:00080000:1.0:1767319936.220094:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319936.220097:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319936.220110:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9880 x1853163985298560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319936.220199:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985298688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319936.220356:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985298432:0@lo:400:kworker.0 00000100:00100000:3.0:1767319936.220419:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985298176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319936.220488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985297920:0@lo:38:kworker.0 00000100:00100000:2.0:1767319936.220826:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319936.220932:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298432 00000100:00100000:2.0:1767319936.220941:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298432 00000100:00100000:2.0:1767319936.220999:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4920 00000100:00100000:2.0:1767319936.221009:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985298432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319936.221063:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985298432:12345-0@lo:400:kworker.0 Request processed in 55us (237us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319936.221070:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4920 00000100:00080000:2.0:1767319936.221092:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319936.223032:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985298432:0@lo:400:kworker.0 00000100:00100000:2.0:1767319936.223104:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985298816:0@lo:400:kworker.0 00000100:00100000:2.0:1767319936.223352:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319936.223414:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298816 00000100:00100000:2.0:1767319936.223421:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298816 00000100:00100000:2.0:1767319936.223559:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3221 00000100:00100000:2.0:1767319936.223568:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985298816:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319936.223735:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319936.223763:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319936.224020:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985298816:12345-0@lo:400:kworker.0 Request processed in 453us (668us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319936.224029:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3221 00000100:00100000:2.0:1767319936.224071:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985298816:0@lo:400:kworker.0 00000100:00100000:2.0:1767319936.224091:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298176 00000100:00100000:2.0:1767319936.224095:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298176 00000100:00080000:3.0:1767319936.224557:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319936.224716:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4921 00000100:00100000:2.0:1767319936.224727:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985298176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319936.224775:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985298176:12345-0@lo:400:kworker.0 Request processed in 50us (1011us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319936.224785:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4921 00000100:00100000:0.0:1767319936.226656:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319936.227103:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319936.227177:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298688 00000100:00100000:1.0:1767319936.227185:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298688 00000100:00100000:1.0:1767319936.227234:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3222 00000100:00100000:1.0:1767319936.232400:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985298688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319936.232534:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985298688:12345-0@lo:400:kworker.0 Request processed in 5288us (5427us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319936.232545:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3222 00000100:00100000:1.0:1767319936.232655:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985298688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319936.240087:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985298048:0@lo:38:kworker.0 00000100:00100000:0.0:1767319936.240180:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319936.240209:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985298304:0@lo:38:kworker.0 00000100:00100000:0.0:1767319936.240233:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319936.245690:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985297920 00000100:00100000:3.0:1767319936.245707:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985297920 00000100:00100000:3.0:1767319936.245770:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5498 00000100:00100000:3.0:1767319936.245779:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985297920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319936.245813:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319936.245892:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985297920:12345-0@lo:38:kworker.0 Request processed in 113us (19241us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319936.246384:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5498 00000100:00080000:3.0:1767319936.246426:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319936.246572:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298048 00000100:00100000:3.0:1767319936.246586:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298048 00000100:00100000:3.0:1767319936.246618:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1865 00000100:00100000:3.0:1767319936.246624:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985298048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319936.246641:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319936.246679:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985298048:12345-0@lo:38:kworker.0 Request processed in 54us (6499us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319936.246686:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1865 00000100:00100000:3.0:1767319936.246754:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985298176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319936.253804:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985298560:0@lo:38:kworker.0 00000100:00100000:0.0:1767319936.253877:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319936.253943:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319936.253948:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319936.253952:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985297920:0@lo:38:kworker.0 00000100:00080000:0.0:1767319936.253963:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319936.253966:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319936.253968:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985298048:0@lo:38:kworker.0 00000100:00100000:1.0:1767319936.254063:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298304 00000100:00100000:1.0:1767319936.254072:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298304 00000100:00100000:1.0:1767319936.254111:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298560 00000100:00100000:1.0:1767319936.254115:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298560 00000100:00100000:1.0:1767319936.254153:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5499 00000100:00100000:1.0:1767319936.254161:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985298304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319936.254200:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319936.254255:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985298304:12345-0@lo:38:kworker.0 Request processed in 93us (14020us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319936.254263:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5499 00000100:00100000:1.0:1767319936.254279:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5500 00000100:00100000:1.0:1767319936.254284:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985298560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319936.254293:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319936.254318:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985298560:12345-0@lo:38:kworker.0 Request processed in 32us (443us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319936.254323:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5500 00000100:00080000:0.0:1767319936.254911:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319936.254918:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319936.254922:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985298304:0@lo:38:kworker.0 00000100:00080000:0.0:1767319936.254933:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319936.254936:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319936.254938:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985298560:0@lo:38:kworker.0 00010000:00010000:1.0:1767319936.859673:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319937.885019:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319938.907669:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319939.803524:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985298944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319939.803585:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319939.803687:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985298944 00000100:00100000:3.0:1767319939.803694:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985298944 00000100:00100000:3.0:1767319939.803752:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2400 00000100:00100000:3.0:1767319939.803759:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985298944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319939.803784:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319939.803817:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985298944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (232us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319939.803821:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2400 00000100:00080000:1.0:1767319939.803953:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319939.803983:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985298944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319939.804129:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319939.804136:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985299072:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319939.804157:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319939.804318:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985299072 00000100:00100000:3.0:1767319939.804322:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985299072 00000100:00100000:3.0:1767319939.804353:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2401 00000100:00100000:3.0:1767319939.804357:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985299072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319939.804370:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319939.804392:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985299072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 35us (234us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319939.804396:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2401 00000100:00080000:1.0:1767319939.804477:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319939.804500:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985299072:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319939.931801:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767319940.266196:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920588928 offset 0 length 520: 4 00000100:00100000:2.0:1767319940.266917:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319940.266961:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319940.267019:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589056 00000100:00100000:3.0:1767319940.267042:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589056 00000100:00100000:3.0:1767319940.267090:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589184 00000100:00100000:3.0:1767319940.267093:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589184 00000100:00100000:3.0:1767319940.267132:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4922 00000100:00100000:3.0:1767319940.267154:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920589184:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767319940.267189:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5501 00000100:00100000:1.0:1767319940.267198:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920589056:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767319940.267220:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.267264:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920589184:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 122us (300us total) trans 0 rc 0/0 00000100:00080000:2.0:1767319940.267269:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319940.267274:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4922 00000100:00100000:1.0:1767319940.267803:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920589056:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 603us (886us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319940.267811:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5501 00000100:00100000:2.0:1767319940.379319:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319915/real 1767319915] req@ffff8ce1b5b7ed80 x1853163985291904/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319940 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319940.379356:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5b7ed80 x1853163985291904/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319940 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319940.379376:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319940.379413:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319940.379417:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985291904:0@lo:250:kworker.0 00000100:00080000:1.0:1767319940.379572:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319940.379578:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319940.379581:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319940.379585:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5164 00000100:00080000:1.0:1767319940.379589:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319940.379594:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319940.379625:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985299200/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319940.379652:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319940.379656:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319940.379658:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319940.379662:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5184 00000100:00080000:1.0:1767319940.379664:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319940.379667:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319940.379683:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4380 x1853163985299328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319940.379692:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319940.379695:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319940.379697:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319940.379700:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5184 00000100:00080000:1.0:1767319940.379703:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319940.379706:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319940.379721:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593bb80 x1853163985299456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319940.379777:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319940.379781:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319940.379783:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319940.379787:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5184 00000100:00080000:1.0:1767319940.379790:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319940.379794:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319940.379804:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593aa00 x1853163985299712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319940.379819:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319940.379821:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319940.379823:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319940.379827:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5184 00000100:00080000:1.0:1767319940.379829:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319940.379832:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319940.379842:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939500 x1853163985299968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319940.379928:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985299584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319940.379968:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985299200:0@lo:250:kworker.0 00000400:00000100:2.0:1767319940.380008:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985299200 offset 0 length 520: 4 00000100:00100000:3.0:1767319940.380015:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319940.380049:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985299328:0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.380053:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319940.380063:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985299840:0@lo:400:kworker.0 00000100:00100000:2.0:1767319940.380076:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319940.380109:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319940.380111:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985299456:0@lo:38:kworker.0 00000100:00100000:2.0:1767319940.380150:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319940.380192:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985299712:0@lo:38:kworker.0 00000100:00100000:2.0:1767319940.380212:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319940.380228:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985299968:0@lo:38:kworker.0 00000100:00100000:2.0:1767319940.380246:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319940.380281:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985299584 00000100:00100000:2.0:1767319940.380288:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985299584 00000100:00100000:0.0:1767319940.380309:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985299456 00000100:00100000:0.0:1767319940.380316:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985299456 00000100:00100000:3.0:1767319940.380322:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985299328 00000100:00100000:3.0:1767319940.380326:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985299328 00000100:00100000:2.0:1767319940.380342:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985299840 00000100:00100000:3.0:1767319940.380344:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985299712 00000100:00100000:0.0:1767319940.380347:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1866 00000100:00100000:2.0:1767319940.380348:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985299840 00000100:00100000:3.0:1767319940.380349:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985299712 00000100:00100000:0.0:1767319940.380354:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985299456:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.380360:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985299968 00000100:00100000:3.0:1767319940.380362:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985299968 00010000:00020000:0.0:1767319940.380375:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319940.380418:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985299456:12345-0@lo:38:kworker.0 Request processed in 63us (267us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319940.380426:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1866 00000100:00100000:2.0:1767319940.380443:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4923 00000100:00100000:2.0:1767319940.380455:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985299584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319940.380506:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985299584:12345-0@lo:400:kworker.0 Request processed in 55us (494us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319940.381827:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4924 00000100:00100000:3.0:1767319940.381844:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985299840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319940.381907:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985299840:12345-0@lo:400:kworker.0 Request processed in 71us (1798us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319940.381914:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4924 00000100:00100000:3.0:1767319940.381941:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5502 00000100:00100000:3.0:1767319940.381962:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985299328:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319940.381980:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.382005:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985299328:12345-0@lo:38:kworker.0 Request processed in 43us (1931us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319940.382009:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5502 00000100:00100000:3.0:1767319940.382019:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5503 00000100:00100000:3.0:1767319940.382023:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985299712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319940.382032:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.382053:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985299712:12345-0@lo:38:kworker.0 Request processed in 29us (1840us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319940.382059:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5503 00000100:00100000:3.0:1767319940.382069:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5504 00000100:00100000:3.0:1767319940.382073:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985299968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319940.382082:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.382098:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985299968:12345-0@lo:38:kworker.0 Request processed in 25us (1853us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319940.382102:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5504 00000100:00080000:3.0:1767319940.382397:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319940.382449:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.382454:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.382458:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985299328:0@lo:38:kworker.0 00000100:00080000:3.0:1767319940.382469:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.382472:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.382475:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985299456:0@lo:38:kworker.0 00000100:00080000:3.0:1767319940.382483:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.382486:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.382488:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985299712:0@lo:38:kworker.0 00000100:00080000:3.0:1767319940.382494:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.382497:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.382500:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985299968:0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.382540:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985299840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319940.382550:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985299584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319940.391233:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4923 00010000:00010000:1.0:1767319940.955454:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319941.979766:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319943.003486:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319944.028053:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319944.926252:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985300224:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319944.926327:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319944.926413:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319944.926421:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985300096:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319944.926444:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319944.927675:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300224 00000100:00100000:0.0:1767319944.927688:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300224 00000100:00100000:0.0:1767319944.927823:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300096 00000100:00100000:0.0:1767319944.927827:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300096 00000100:00100000:1.0:1767319944.927890:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2402 00000100:00100000:1.0:1767319944.927903:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985300224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319944.927936:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319944.928003:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985300224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (1676us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319944.928013:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2402 00000100:00100000:1.0:1767319944.928064:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985300224:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319944.930408:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319944.930422:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319944.930762:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2403 00000100:00100000:0.0:1767319944.930777:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985300096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319944.930812:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319944.930888:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985300096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 116us (4441us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319944.930895:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2403 00000100:00100000:1.0:1767319944.931004:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985300096:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319945.052640:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319945.386264:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319945.386310:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319945.388915:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589440 00000100:00100000:2.0:1767319945.388934:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589440 00000100:00100000:2.0:1767319945.389184:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589312 00000100:00100000:2.0:1767319945.389189:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589312 00000100:00100000:2.0:1767319945.389213:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4925 00000100:00100000:2.0:1767319945.389225:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920589440:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319945.389312:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920589440:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (3060us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319945.389320:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4925 00000100:00100000:2.0:1767319945.389356:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5505 00000100:00100000:2.0:1767319945.389360:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920589312:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319945.389383:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319945.389737:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319945.415925:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920589312:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 26549us (29601us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319945.415941:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5505 00000100:00080000:1.0:1767319945.435400:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319945.435412:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319945.435416:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319945.435420:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319945.435425:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319945.435435:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5188 00000100:00080000:1.0:1767319945.435440:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319945.435447:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319945.435487:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a680 x1853163985300352/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319945.435515:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319945.435519:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319945.435520:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319945.435524:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5188 00000100:00080000:1.0:1767319945.435526:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319945.435530:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319945.435541:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985300480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319945.435632:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985300608:0@lo:400:kworker.0 00000100:00100000:2.0:1767319945.435699:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319945.435812:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985300352:0@lo:38:kworker.0 00000100:00100000:3.0:1767319945.435841:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319945.435861:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985300480:0@lo:38:kworker.0 00000100:00100000:3.0:1767319945.435919:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319945.435960:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300480 00000100:00100000:3.0:1767319945.435968:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300480 00000100:00100000:3.0:1767319945.436002:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1867 00000100:00100000:3.0:1767319945.436009:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190db9050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985300480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319945.436030:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319945.436552:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300608 00000100:00100000:0.0:1767319945.436558:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300608 00000100:00100000:0.0:1767319945.436606:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4926 00000100:00100000:0.0:1767319945.436614:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985300608:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319945.436748:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190db9050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985300480:12345-0@lo:38:kworker.0 Request processed in 737us (827us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319945.436756:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1867 00000100:00080000:3.0:1767319945.436953:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319945.436964:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300352 00000100:00100000:3.0:1767319945.436968:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300352 00000100:00100000:3.0:1767319945.436986:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5506 00000100:00100000:3.0:1767319945.436990:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985300352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319945.437003:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319945.437027:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985300352:12345-0@lo:38:kworker.0 Request processed in 35us (1185us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319945.437045:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5506 00000100:00080000:3.0:1767319945.437081:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319945.437087:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319945.437090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985300352:0@lo:38:kworker.0 00000100:00080000:3.0:1767319945.437099:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319945.437103:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319945.437106:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985300480:0@lo:38:kworker.0 00000100:00080000:1.0:1767319945.437283:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319945.437291:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319945.437293:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319945.437298:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5188 00000100:00080000:1.0:1767319945.437301:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319945.437306:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319945.437328:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938700 x1853163985300736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319945.437986:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985300864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319945.438032:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319945.438086:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300864 00000100:00100000:3.0:1767319945.438092:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300864 00000100:00100000:3.0:1767319945.438128:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4927 00000100:00100000:3.0:1767319945.438135:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985300864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319945.438191:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985300864:12345-0@lo:400:kworker.0 Request processed in 56us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319945.438199:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4927 00000100:00080000:3.0:1767319945.438214:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319945.438241:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985300864:0@lo:400:kworker.0 00000100:00100000:2.0:1767319945.438604:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985300736:0@lo:38:kworker.0 00000100:00100000:2.0:1767319945.438748:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319945.438783:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300736 00000100:00100000:2.0:1767319945.438787:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300736 00000100:00100000:2.0:1767319945.438822:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5507 00000100:00100000:2.0:1767319945.438828:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985300736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319945.438844:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319945.438877:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985300736:12345-0@lo:38:kworker.0 Request processed in 48us (129us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319945.438884:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5507 00000100:00080000:2.0:1767319945.438910:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319945.438914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319945.438917:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985300736:0@lo:38:kworker.0 00000100:00100000:0.0:1767319945.439094:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985300608:12345-0@lo:400:kworker.0 Request processed in 2479us (3392us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319945.439103:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4926 00000100:00080000:1.0:1767319945.439268:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319945.439273:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319945.439276:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319945.439280:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5188 00000100:00080000:1.0:1767319945.439284:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319945.439288:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319945.439312:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939f80 x1853163985300992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319945.439464:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985300992:0@lo:38:kworker.0 00000100:00100000:2.0:1767319945.439493:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.439565:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985300992 00000100:00100000:0.0:1767319945.439570:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985300992 00000100:00100000:0.0:1767319945.439594:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5508 00000100:00100000:0.0:1767319945.439599:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985300992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319945.439612:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319945.439774:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985300992:12345-0@lo:38:kworker.0 Request processed in 173us (280us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319945.439781:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5508 00000100:00080000:2.0:1767319945.439909:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319945.439913:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319945.439917:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985300992:0@lo:38:kworker.0 00000100:00100000:3.0:1767319945.440006:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985300608:0@lo:400:kworker.0 00010000:00010000:0.0:1767319946.079659:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319947.099718:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319948.123394:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319949.147483:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319950.044552:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319950.044580:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985301120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319950.044649:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.045511:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985301248:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319950.045560:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319950.045761:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985301120 00000100:00100000:3.0:1767319950.045768:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985301120 00000100:00100000:3.0:1767319950.045830:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985301248 00000100:00100000:3.0:1767319950.045833:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985301248 00000100:00100000:3.0:1767319950.045857:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2404 00000100:00100000:3.0:1767319950.045866:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985301120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319950.045893:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319950.045948:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985301120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (1286us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319950.045957:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2404 00000100:00100000:3.0:1767319950.045973:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2405 00000100:00100000:3.0:1767319950.045979:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985301248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319950.045994:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319950.046142:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985301120:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319950.046243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319950.046259:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319950.046374:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985301248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 394us (812us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319950.046381:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2405 00000100:00100000:2.0:1767319950.046454:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985301248:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319950.171444:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319950.507562:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319950.507766:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319950.513522:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589696 00000100:00100000:0.0:1767319950.513550:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589696 00000100:00080000:1.0:1767319950.513719:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319950.513776:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589568 00000100:00100000:0.0:1767319950.513781:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589568 00000100:00100000:2.0:1767319950.513834:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4928 00000100:00100000:2.0:1767319950.513853:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920589696:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319950.513979:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920589696:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 135us (6432us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319950.513988:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4928 00000100:00100000:2.0:1767319950.514012:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5509 00000100:00100000:2.0:1767319950.514016:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920589568:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319950.514042:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319950.514069:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920589568:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 50us (6303us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319950.514076:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5509 00000100:00080000:1.0:1767319950.555349:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.555362:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319950.555366:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.555372:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.555378:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319950.555389:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.555395:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.555404:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319950.555465:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939180 x1853163985301376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319950.555504:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.555508:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.555510:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319950.555523:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.555525:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.555529:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319950.555544:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939880 x1853163985301504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319950.555583:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.555587:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.555589:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319950.555593:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.555595:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.555598:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319950.555610:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a300 x1853163985301760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319950.555626:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.555630:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.555631:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319950.555634:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.555637:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.555639:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319950.555649:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938700 x1853163985302016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319950.555795:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985301888:0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.555884:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319950.555964:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319950.555970:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985301632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.555995:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319950.556038:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985301888 00000100:00100000:3.0:1767319950.556046:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985301888 00000100:00100000:3.0:1767319950.556109:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985301632 00000100:00100000:3.0:1767319950.556112:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985301632 00000100:00100000:3.0:1767319950.556140:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4929 00000100:00100000:3.0:1767319950.556150:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985301888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.556228:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985301888:12345-0@lo:400:kworker.0 Request processed in 79us (343us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319950.556237:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4929 00000100:00100000:3.0:1767319950.556272:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4930 00000100:00100000:3.0:1767319950.556279:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985301632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.556313:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985301632:12345-0@lo:400:kworker.0 Request processed in 34us (316us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319950.556323:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4930 00000100:00080000:3.0:1767319950.556349:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319950.556356:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985301376:0@lo:38:kworker.0 00000100:00100000:3.0:1767319950.556387:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985301632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.556400:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985301888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319950.556465:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.556498:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985301504:0@lo:38:kworker.0 00000100:00100000:2.0:1767319950.556534:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.556566:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985301760:0@lo:38:kworker.0 00000100:00100000:2.0:1767319950.556591:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.556608:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985302016:0@lo:38:kworker.0 00000100:00100000:2.0:1767319950.556629:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.556655:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985301376 00000100:00100000:2.0:1767319950.556659:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985301376 00000100:00100000:2.0:1767319950.556678:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985301760 00000100:00100000:2.0:1767319950.556681:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985301760 00000100:00100000:2.0:1767319950.556690:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302016 00000100:00100000:2.0:1767319950.556692:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302016 00000100:00100000:2.0:1767319950.556704:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5510 00000100:00100000:2.0:1767319950.556710:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985301376:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319950.556718:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5511 00000100:00100000:1.0:1767319950.556726:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985301760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319950.556734:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767319950.556739:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319950.556779:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985301376:12345-0@lo:38:kworker.0 Request processed in 68us (312us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319950.556787:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985301760:12345-0@lo:38:kworker.0 Request processed in 59us (193us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319950.556788:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5510 00000100:00100000:1.0:1767319950.556800:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5511 00000100:00100000:2.0:1767319950.556811:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5512 00000100:00100000:2.0:1767319950.556817:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985302016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319950.556828:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319950.556854:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985302016:12345-0@lo:38:kworker.0 Request processed in 36us (225us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319950.556861:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5512 00000100:00100000:3.0:1767319950.556889:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985301504 00000100:00080000:2.0:1767319950.556894:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319950.556895:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985301504 00000100:00080000:2.0:1767319950.556903:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319950.556908:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985301376:0@lo:38:kworker.0 00000100:00080000:2.0:1767319950.556932:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319950.556938:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319950.556941:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985301760:0@lo:38:kworker.0 00000100:00080000:2.0:1767319950.556950:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319950.556953:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319950.556956:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985302016:0@lo:38:kworker.0 00000100:00100000:3.0:1767319950.556973:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1868 00000100:00100000:3.0:1767319950.556979:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985301504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319950.556991:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319950.557027:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985301504:12345-0@lo:38:kworker.0 Request processed in 46us (491us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319950.557035:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319950.557035:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1868 00000100:00080000:2.0:1767319950.557043:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319950.557049:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985301504:0@lo:38:kworker.0 00010000:00010000:0.0:1767319951.195794:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319952.220497:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319953.243515:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319954.267622:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319955.163538:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985302144:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319955.163616:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319955.163757:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302144 00000100:00100000:0.0:1767319955.163774:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302144 00000100:00100000:0.0:1767319955.163849:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2406 00000100:00100000:0.0:1767319955.163858:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985302144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319955.163888:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319955.163938:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985302144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (322us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319955.163945:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2406 00000100:00080000:1.0:1767319955.164003:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319955.164039:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985302144:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319955.164421:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319955.167016:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985302272:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319955.167094:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.167616:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302272 00000100:00100000:3.0:1767319955.167629:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302272 00000100:00100000:3.0:1767319955.167735:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2407 00000100:00100000:3.0:1767319955.167746:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985302272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319955.168045:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319955.168116:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985302272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 372us (1021us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319955.168124:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2407 00000100:00100000:2.0:1767319955.168238:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985302272:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319955.168363:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319955.291538:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319955.629780:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319955.629880:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319955.630215:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589952 00000100:00100000:3.0:1767319955.630225:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589952 00000100:00100000:3.0:1767319955.630296:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920589824 00000100:00100000:3.0:1767319955.630299:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920589824 00000100:00100000:3.0:1767319955.630323:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4931 00000100:00100000:3.0:1767319955.630332:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920589952:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767319955.630417:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920589952:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (645us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319955.630425:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4931 00000100:00100000:3.0:1767319955.630456:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5513 00000100:00100000:3.0:1767319955.630460:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920589824:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767319955.630481:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319955.630506:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920589824:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (627us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319955.630514:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5513 00000100:00080000:3.0:1767319955.630553:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319955.675421:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319955.675439:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319955.675442:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319955.675446:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319955.675450:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319955.675459:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5198 00000100:00080000:0.0:1767319955.675465:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319955.675471:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319955.675511:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7ca80 x1853163985302400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319955.675528:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319955.675530:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319955.675532:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319955.675535:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5198 00000100:00080000:0.0:1767319955.675537:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319955.675541:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319955.675552:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7e300 x1853163985302528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319955.675588:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319955.675591:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319955.675593:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319955.675596:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5198 00000100:00080000:0.0:1767319955.675599:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319955.675602:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319955.675612:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7c380 x1853163985302784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319955.675626:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319955.675629:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319955.675631:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319955.675634:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5198 00000100:00080000:0.0:1767319955.675636:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319955.675638:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319955.675650:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7dc00 x1853163985303040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319955.675805:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985302400:0@lo:38:kworker.0 00000100:00100000:2.0:1767319955.675874:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319955.675948:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985302528:0@lo:38:kworker.0 00000100:00100000:2.0:1767319955.675977:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319955.675987:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985302656:0@lo:400:kworker.0 00000100:00100000:2.0:1767319955.676019:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985302784:0@lo:38:kworker.0 00000100:00100000:1.0:1767319955.676026:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319955.676052:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319955.676055:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319955.676064:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985302912:0@lo:400:kworker.0 00000100:00100000:2.0:1767319955.676077:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985303040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319955.676090:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319955.676109:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319955.676141:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302528 00000100:00100000:2.0:1767319955.676159:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302400 00000100:00100000:2.0:1767319955.676186:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302400 00000100:00100000:2.0:1767319955.676218:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302656 00000100:00100000:2.0:1767319955.676221:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302656 00000100:00100000:2.0:1767319955.676253:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302784 00000100:00100000:2.0:1767319955.676255:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302784 00000100:00100000:2.0:1767319955.676265:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4932 00000100:00100000:2.0:1767319955.676273:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985302656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319955.676323:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985302656:12345-0@lo:400:kworker.0 Request processed in 50us (298us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319955.676330:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4932 00000100:00100000:2.0:1767319955.676346:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985302912 00000100:00100000:2.0:1767319955.676349:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302912 00000100:00100000:2.0:1767319955.676364:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303040 00000100:00100000:2.0:1767319955.676367:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303040 00000100:00100000:2.0:1767319955.676379:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4933 00000100:00100000:2.0:1767319955.676385:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985302912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319955.676412:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985302912:12345-0@lo:400:kworker.0 Request processed in 27us (323us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319955.676418:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4933 00000100:00100000:2.0:1767319955.676441:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5514 00000100:00100000:2.0:1767319955.676446:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985302400:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319955.676464:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319955.676496:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985302400:12345-0@lo:38:kworker.0 Request processed in 48us (622us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319955.676502:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5514 00000100:00100000:2.0:1767319955.676514:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5515 00000100:00100000:2.0:1767319955.676518:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985302784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319955.676527:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319955.676548:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985302784:12345-0@lo:38:kworker.0 Request processed in 28us (496us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319955.676553:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5515 00000100:00100000:2.0:1767319955.676565:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5516 00000100:00100000:2.0:1767319955.676569:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985303040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319955.676578:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319955.676598:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985303040:12345-0@lo:38:kworker.0 Request processed in 28us (490us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319955.676605:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5516 00000100:00080000:2.0:1767319955.676624:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319955.676651:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985302912:0@lo:400:kworker.0 00000100:00100000:2.0:1767319955.676663:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985302656:0@lo:400:kworker.0 00000100:00080000:2.0:1767319955.676692:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319955.676697:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319955.676701:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985302400:0@lo:38:kworker.0 00000100:00080000:2.0:1767319955.676709:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319955.676712:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319955.676714:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985302784:0@lo:38:kworker.0 00000100:00080000:2.0:1767319955.676721:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319955.676724:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319955.676727:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985303040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319955.677603:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985302528 00000100:00100000:1.0:1767319955.677646:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1869 00000100:00100000:1.0:1767319955.677663:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985302528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319955.677677:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319955.677731:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985302528:12345-0@lo:38:kworker.0 Request processed in 78us (1753us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319955.677738:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1869 00000100:00080000:2.0:1767319955.678279:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319955.678285:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319955.678289:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985302528:0@lo:38:kworker.0 00010000:00010000:2.0:1767319956.323046:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319957.339575:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319958.366204:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319959.387443:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319960.283503:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319960.283527:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985303168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319960.283592:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319960.283908:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303168 00000100:00100000:3.0:1767319960.283917:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303168 00000100:00100000:3.0:1767319960.283992:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2408 00000100:00100000:3.0:1767319960.284001:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985303168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319960.284037:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319960.284095:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985303168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (503us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319960.284102:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2408 00000100:00080000:2.0:1767319960.284277:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319960.284548:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985303168:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319960.284733:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319960.284744:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985303296:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319960.284775:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.284842:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303296 00000100:00100000:0.0:1767319960.284847:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303296 00000100:00100000:0.0:1767319960.284891:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2409 00000100:00100000:0.0:1767319960.284898:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985303296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319960.284915:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319960.284947:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985303296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 50us (171us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319960.284953:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2409 00000100:00100000:0.0:1767319960.284980:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985303296:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319960.285363:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319960.411499:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319960.746716:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319960.746826:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920590208 00000100:00100000:0.0:1767319960.746836:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920590208 00000100:00100000:0.0:1767319960.746913:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4934 00000100:00100000:0.0:1767319960.746924:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920590208:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319960.747021:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920590208:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (313us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319960.747029:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4934 00000100:00080000:2.0:1767319960.747108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319960.747309:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319960.747409:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920590080 00000100:00100000:2.0:1767319960.747413:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920590080 00000100:00100000:2.0:1767319960.747440:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5517 00000100:00100000:2.0:1767319960.747445:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920590080:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319960.747461:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319960.747489:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920590080:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (180us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319960.747495:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5517 00000100:00080000:1.0:1767319960.795965:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319960.795972:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319960.795975:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319960.795979:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319960.795982:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319960.795989:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5203 00000100:00080000:1.0:1767319960.795993:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319960.795999:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319960.796029:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939500 x1853163985303424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319960.796062:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319960.796065:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319960.796067:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319960.796071:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5203 00000100:00080000:1.0:1767319960.796073:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319960.796076:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319960.796088:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593aa00 x1853163985303552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319960.796114:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319960.796117:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319960.796119:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319960.796123:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5203 00000100:00080000:1.0:1767319960.796135:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319960.796139:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319960.796150:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939880 x1853163985303808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319960.796184:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319960.796187:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319960.796189:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319960.796193:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5203 00000100:00080000:1.0:1767319960.796195:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319960.796198:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319960.796210:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a300 x1853163985304064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319960.796254:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985304320:0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796316:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319960.796371:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319960.796377:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985304192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796399:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319960.796426:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985304320 00000100:00100000:1.0:1767319960.796433:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985304320 00000100:00100000:1.0:1767319960.796469:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985304192 00000100:00100000:1.0:1767319960.796473:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985304192 00000100:00100000:1.0:1767319960.796495:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3223 00000100:00100000:1.0:1767319960.796503:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985304320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796550:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985304320:12345-0@lo:400:kworker.0 Request processed in 48us (233us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319960.796558:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3223 00000100:00100000:1.0:1767319960.796573:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3224 00000100:00100000:1.0:1767319960.796580:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985304192:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796603:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985304192:12345-0@lo:400:kworker.0 Request processed in 25us (204us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319960.796608:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3224 00000100:00100000:1.0:1767319960.796638:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985304192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796647:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985304320:0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796689:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985303936:0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796713:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319960.796737:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319960.796742:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985303680:0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796807:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319960.796837:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303936 00000100:00100000:1.0:1767319960.796841:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303936 00000100:00100000:1.0:1767319960.796867:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303680 00000100:00100000:1.0:1767319960.796870:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303680 00000100:00100000:1.0:1767319960.796886:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4935 00000100:00100000:1.0:1767319960.796891:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985303936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796924:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985303936:12345-0@lo:400:kworker.0 Request processed in 33us (210us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319960.796929:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4935 00000100:00100000:1.0:1767319960.796940:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4936 00000100:00100000:1.0:1767319960.796945:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985303680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.796969:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985303680:12345-0@lo:400:kworker.0 Request processed in 24us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319960.796975:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4936 00000100:00100000:1.0:1767319960.796994:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985303680:0@lo:400:kworker.0 00000100:00100000:1.0:1767319960.797002:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985303936:0@lo:400:kworker.0 00000100:00100000:0.0:1767319960.797515:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985303424:0@lo:38:kworker.0 00000100:00100000:0.0:1767319960.797549:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.797583:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985303552:0@lo:38:kworker.0 00000100:00100000:0.0:1767319960.797607:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.797630:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985303808:0@lo:38:kworker.0 00000100:00100000:0.0:1767319960.797648:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.797661:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985304064:0@lo:38:kworker.0 00000100:00100000:0.0:1767319960.797679:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.797711:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303552 00000100:00100000:0.0:1767319960.797715:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303552 00000100:00100000:0.0:1767319960.797738:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1870 00000100:00100000:0.0:1767319960.797743:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985303552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319960.797759:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319960.797787:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985303552:12345-0@lo:38:kworker.0 Request processed in 43us (179us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319960.797792:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1870 00000100:00100000:0.0:1767319960.797813:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303424 00000100:00100000:0.0:1767319960.797817:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303424 00000100:00100000:0.0:1767319960.797830:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985303808 00000100:00100000:0.0:1767319960.797833:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985303808 00000100:00100000:0.0:1767319960.797840:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985304064 00000100:00100000:0.0:1767319960.797842:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985304064 00000100:00100000:0.0:1767319960.797850:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5518 00000100:00100000:0.0:1767319960.797853:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985303424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319960.797861:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319960.797882:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985303424:12345-0@lo:38:kworker.0 Request processed in 28us (333us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319960.797887:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5518 00000100:00100000:0.0:1767319960.797897:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5519 00000100:00100000:0.0:1767319960.797900:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985303808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319960.797906:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319960.797924:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985303808:12345-0@lo:38:kworker.0 Request processed in 23us (275us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319960.797928:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5519 00000100:00100000:0.0:1767319960.797945:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5520 00000100:00100000:0.0:1767319960.797948:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985304064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319960.797956:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319960.797976:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985304064:12345-0@lo:38:kworker.0 Request processed in 26us (296us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319960.797982:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5520 00000100:00080000:0.0:1767319960.798004:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319960.798009:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319960.798012:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985303424:0@lo:38:kworker.0 00000100:00080000:0.0:1767319960.798021:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319960.798024:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319960.798027:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985303552:0@lo:38:kworker.0 00000100:00080000:0.0:1767319960.798035:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319960.798038:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319960.798040:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985303808:0@lo:38:kworker.0 00000100:00080000:0.0:1767319960.798047:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319960.798059:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319960.798062:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985304064:0@lo:38:kworker.0 00000100:00080000:2.0:1767319960.798216:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319961.435433:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319962.459631:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319963.483869:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319964.510606:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319965.403495:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319965.403520:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985304448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319965.403591:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.403713:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985304448 00000100:00100000:3.0:1767319965.403721:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985304448 00000100:00100000:3.0:1767319965.403799:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2410 00000100:00100000:3.0:1767319965.403810:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985304448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319965.403838:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319965.403881:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985304448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (290us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319965.403889:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2410 00000100:00080000:0.0:1767319965.404338:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319965.404444:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985304576:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319965.404475:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319965.404530:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985304448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319965.404642:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985304576 00000100:00100000:0.0:1767319965.404647:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985304576 00000100:00100000:0.0:1767319965.404695:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2411 00000100:00100000:0.0:1767319965.404702:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985304576:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319965.404721:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767319965.404758:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985304576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 57us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319965.404764:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2411 00000100:00080000:2.0:1767319965.404819:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319965.404921:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985304576:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319965.533236:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319965.871529:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319965.871582:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319965.871689:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920590464 00000100:00100000:1.0:1767319965.871700:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920590464 00000100:00100000:1.0:1767319965.871770:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920590336 00000100:00100000:1.0:1767319965.871774:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920590336 00000100:00100000:1.0:1767319965.871792:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4937 00000100:00100000:1.0:1767319965.871803:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920590464:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319965.871860:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5521 00000100:00100000:2.0:1767319965.871869:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920590336:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767319965.871881:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920590464:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (362us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319965.871889:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4937 00010000:00020000:2.0:1767319965.871895:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319965.871928:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920590336:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 59us (345us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319965.871935:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5521 00000100:00080000:3.0:1767319965.873525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767319965.874931:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920590592 offset 0 length 520: 4 00000100:00100000:0.0:1767319965.915681:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319940/real 1767319940] req@ffff8ce19e9c4700 x1853163985299200/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319965 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319965.915719:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19e9c4700 x1853163985299200/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319965 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319965.915742:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319965.915763:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319965.915768:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985299200:0@lo:250:kworker.0 00000100:00080000:3.0:1767319965.916043:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319965.916049:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319965.916052:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319965.916057:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5188 00000100:00080000:3.0:1767319965.916061:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319965.916067:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319965.916095:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18f100 x1853163985304704/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319965.916120:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319965.916124:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319965.916125:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319965.916129:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5209 00000100:00080000:3.0:1767319965.916131:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319965.916135:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319965.916151:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3480 x1853163985304832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319965.916160:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319965.916181:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319965.916182:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319965.916186:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5209 00000100:00080000:3.0:1767319965.916188:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319965.916192:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319965.916203:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3b80 x1853163985304960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319965.916257:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319965.916261:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319965.916262:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319965.916266:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5209 00000100:00080000:3.0:1767319965.916268:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319965.916271:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319965.916283:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba4000 x1853163985305216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319965.916298:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319965.916301:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319965.916302:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319965.916305:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5209 00000100:00080000:3.0:1767319965.916307:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319965.916309:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319965.916319:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d500 x1853163985305472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319965.916369:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985305600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.916432:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.916489:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319965.916495:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985305728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.916519:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.916546:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305600 00000100:00100000:3.0:1767319965.916553:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305600 00000100:00100000:3.0:1767319965.916606:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305728 00000100:00100000:3.0:1767319965.916610:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305728 00000100:00100000:3.0:1767319965.916630:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3225 00000100:00100000:3.0:1767319965.916637:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985305600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.916686:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985305600:12345-0@lo:400:kworker.0 Request processed in 49us (253us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319965.916693:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3225 00000100:00100000:3.0:1767319965.916709:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3226 00000100:00100000:3.0:1767319965.916715:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a07c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985305728:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.916737:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a07c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985305728:12345-0@lo:400:kworker.0 Request processed in 23us (218us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319965.916743:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3226 00000100:00100000:3.0:1767319965.916938:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985305728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.916947:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985305600:0@lo:400:kworker.0 00000100:00100000:2.0:1767319965.917217:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985305344:0@lo:400:kworker.0 00000100:00100000:2.0:1767319965.917256:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319965.917265:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985305088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319965.917297:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319965.917338:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305344 00000100:00100000:1.0:1767319965.917342:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305344 00000100:00100000:1.0:1767319965.917380:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305088 00000100:00100000:1.0:1767319965.917383:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305088 00000100:00100000:1.0:1767319965.917399:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4938 00000100:00100000:1.0:1767319965.917404:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985305344:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319965.917432:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985305344:12345-0@lo:400:kworker.0 Request processed in 28us (177us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319965.917438:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4938 00000100:00100000:1.0:1767319965.917448:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4939 00000100:00100000:1.0:1767319965.917453:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985305088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319965.917475:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985305088:12345-0@lo:400:kworker.0 Request processed in 22us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319965.917484:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4939 00000100:00100000:1.0:1767319965.917505:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985305088:0@lo:400:kworker.0 00000100:00080000:2.0:1767319965.917649:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319965.917676:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985305344:0@lo:400:kworker.0 00000100:00100000:0.0:1767319965.917771:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985304704:0@lo:250:kworker.0 00000400:00000100:0.0:1767319965.917795:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985304704 offset 0 length 520: 4 00000100:00100000:0.0:1767319965.917812:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985304832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.917831:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319965.917853:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985304960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.917873:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319965.917892:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985305216:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.917910:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319965.917926:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985305472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.917943:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319965.917977:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985304832 00000100:00100000:0.0:1767319965.917981:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985304832 00000100:00100000:0.0:1767319965.917994:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305216 00000100:00100000:0.0:1767319965.917996:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305216 00000100:00100000:0.0:1767319965.918009:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305472 00000100:00100000:0.0:1767319965.918012:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305472 00000100:00100000:0.0:1767319965.918021:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5522 00000100:00100000:0.0:1767319965.918026:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985304832:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319965.918038:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985304960 00010000:00020000:0.0:1767319965.918044:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319965.918048:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985304960 00000100:00100000:0.0:1767319965.918078:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985304832:12345-0@lo:38:kworker.0 Request processed in 51us (245us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319965.918085:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1871 00000100:00100000:0.0:1767319965.918088:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5522 00000100:00100000:2.0:1767319965.918096:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985304960:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.918104:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5523 00000100:00100000:0.0:1767319965.918109:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985305216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319965.918111:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767319965.918122:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319965.918182:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985304960:12345-0@lo:38:kworker.0 Request processed in 66us (287us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319965.918190:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1871 00000100:00100000:2.0:1767319965.918213:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5524 00000100:00100000:2.0:1767319965.918217:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985305472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319965.918228:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319965.918258:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985305472:12345-0@lo:38:kworker.0 Request processed in 39us (313us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319965.918266:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5524 00000100:00080000:1.0:1767319965.918351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319965.918356:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319965.918360:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985304832:0@lo:38:kworker.0 00000100:00080000:1.0:1767319965.918367:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319965.918370:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319965.918373:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985304960:0@lo:38:kworker.0 00000100:00080000:1.0:1767319965.918379:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319965.918382:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319965.918385:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985305216:0@lo:38:kworker.0 00000100:00080000:1.0:1767319965.918391:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319965.918394:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319965.918396:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985305472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.923504:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985305216:12345-0@lo:38:kworker.0 Request processed in 5383us (5581us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319965.923520:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5523 00010000:00010000:1.0:1767319966.560022:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319967.579355:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319968.603683:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319969.629130:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319970.523910:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985305856:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319970.523981:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319970.524198:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985305984:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319970.524434:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319970.531799:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305856 00000100:00100000:2.0:1767319970.531815:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305856 00000100:00100000:2.0:1767319970.531906:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985305984 00000100:00100000:2.0:1767319970.531910:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985305984 00000100:00100000:2.0:1767319970.531939:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2412 00000100:00100000:2.0:1767319970.531950:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985305856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319970.531984:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319970.532081:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985305856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 121us (8087us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319970.532089:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2412 00000100:00100000:2.0:1767319970.532107:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2413 00000100:00100000:2.0:1767319970.532113:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985305984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319970.532133:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319970.532184:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985305984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (7750us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319970.532190:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2413 00000100:00100000:2.0:1767319970.532252:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985305856:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319970.532362:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985305984:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319970.532721:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319970.532726:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767319970.651478:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319970.992934:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767319970.992991:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319970.993094:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920590848 00000100:00100000:2.0:1767319970.993107:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920590848 00000100:00100000:2.0:1767319970.993227:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920590720 00000100:00100000:2.0:1767319970.993231:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920590720 00000100:00100000:2.0:1767319970.993264:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4940 00000100:00100000:2.0:1767319970.993275:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920590848:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319970.993306:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5525 00000100:00100000:0.0:1767319970.993315:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920590720:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319970.993337:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319970.993347:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920590848:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 72us (420us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319970.993355:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4940 00000100:00100000:0.0:1767319970.993374:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920590720:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 57us (382us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319970.993383:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5525 00000100:00080000:3.0:1767319970.994628:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319971.036839:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319971.036850:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319971.036854:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319971.036858:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319971.036875:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319971.036884:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5214 00000100:00080000:3.0:1767319971.036889:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319971.036897:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319971.036953:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77480 x1853163985306112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319971.036980:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319971.036984:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319971.036986:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319971.036990:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5214 00000100:00080000:3.0:1767319971.036992:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319971.036996:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319971.037009:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77b80 x1853163985306240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319971.037040:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319971.037044:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319971.037046:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319971.037049:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5214 00000100:00080000:3.0:1767319971.037051:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319971.037054:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319971.037066:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985306496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319971.037080:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319971.037083:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319971.037085:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319971.037088:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5214 00000100:00080000:3.0:1767319971.037091:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319971.037094:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319971.037105:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5ba7100 x1853163985306752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319971.037121:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985306624:0@lo:400:kworker.0 00000100:00100000:3.0:1767319971.037202:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985306880:0@lo:400:kworker.0 00000100:00100000:2.0:1767319971.037222:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319971.037414:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985306624 00000100:00100000:1.0:1767319971.037423:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985306624 00000100:00100000:0.0:1767319971.037437:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985306368:0@lo:400:kworker.0 00000100:00100000:1.0:1767319971.037486:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4941 00000100:00100000:1.0:1767319971.037496:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985306624:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319971.038654:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319971.038737:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319971.038747:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985307008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319971.038786:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319971.039321:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985306880 00000100:00100000:3.0:1767319971.039328:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985306880 00000100:00100000:3.0:1767319971.039368:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307008 00000100:00100000:3.0:1767319971.039371:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307008 00000100:00100000:3.0:1767319971.039393:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3227 00000100:00100000:3.0:1767319971.039401:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985306880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319971.039452:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985306880:12345-0@lo:400:kworker.0 Request processed in 51us (802us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319971.039460:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3227 00000100:00100000:3.0:1767319971.039474:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3228 00000100:00100000:3.0:1767319971.039481:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985307008:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319971.039508:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985307008:12345-0@lo:400:kworker.0 Request processed in 28us (722us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319971.039514:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3228 00000100:00100000:3.0:1767319971.039559:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985307008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319971.039570:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985306880:0@lo:400:kworker.0 00000100:00080000:3.0:1767319971.039627:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319971.039643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985306112:0@lo:38:kworker.0 00000100:00100000:3.0:1767319971.039695:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319971.039722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985306240:0@lo:38:kworker.0 00000100:00100000:3.0:1767319971.039747:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319971.039769:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985306496:0@lo:38:kworker.0 00000100:00100000:3.0:1767319971.039789:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319971.039805:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985306752:0@lo:38:kworker.0 00000100:00100000:2.0:1767319971.040203:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985306112 00000100:00100000:2.0:1767319971.040209:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985306112 00000100:00100000:2.0:1767319971.040230:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985306496 00000100:00100000:2.0:1767319971.040232:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985306496 00000100:00100000:2.0:1767319971.040246:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5526 00000100:00100000:2.0:1767319971.040251:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985306112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319971.040268:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319971.040304:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985306112:12345-0@lo:38:kworker.0 Request processed in 52us (609us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319971.040311:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5526 00000100:00100000:2.0:1767319971.040326:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5527 00000100:00100000:2.0:1767319971.040331:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985306496:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319971.040340:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319971.040346:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319971.040376:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985306496:12345-0@lo:38:kworker.0 Request processed in 43us (584us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319971.040383:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5527 00000100:00080000:3.0:1767319971.040384:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319971.040391:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319971.040393:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985306752 00000100:00100000:3.0:1767319971.040397:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985306112:0@lo:38:kworker.0 00000100:00100000:2.0:1767319971.040399:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985306752 00000100:00080000:3.0:1767319971.040414:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767319971.040417:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5528 00000100:00080000:3.0:1767319971.040422:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319971.040428:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985306752:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319971.040430:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985306496:0@lo:38:kworker.0 00010000:00020000:2.0:1767319971.040443:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319971.040463:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985306240 00000100:00100000:3.0:1767319971.040468:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985306240 00000100:00100000:2.0:1767319971.040479:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985306752:12345-0@lo:38:kworker.0 Request processed in 52us (137us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319971.040486:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5528 00000100:00100000:3.0:1767319971.040496:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1872 00000100:00100000:3.0:1767319971.040502:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985306240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319971.040513:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319971.040528:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319971.040534:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319971.040540:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985306752:0@lo:38:kworker.0 00000100:00100000:3.0:1767319971.040546:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985306240:12345-0@lo:38:kworker.0 Request processed in 42us (797us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319971.040555:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1872 00000100:00080000:2.0:1767319971.040562:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319971.040568:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319971.040572:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985306240:0@lo:38:kworker.0 00000100:00100000:1.0:1767319971.041886:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985306624:12345-0@lo:400:kworker.0 Request processed in 4383us (4657us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319971.041898:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4941 00000100:00100000:3.0:1767319971.042034:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985306624:0@lo:400:kworker.0 00000100:00100000:0.0:1767319971.042274:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319971.043299:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985306368 00000100:00100000:1.0:1767319971.043309:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985306368 00000100:00100000:1.0:1767319971.043388:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4942 00000100:00100000:1.0:1767319971.043398:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985306368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319971.043476:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985306368:12345-0@lo:400:kworker.0 Request processed in 80us (1202us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319971.043483:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4942 00000100:00100000:0.0:1767319971.043530:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985306368:0@lo:400:kworker.0 00000100:00080000:3.0:1767319971.043613:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319971.676695:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319972.699869:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319973.726055:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319974.747519:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319975.643868:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319975.643892:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985307136:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319975.644020:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319975.644446:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319975.644458:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985307264:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319975.644490:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319975.644536:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307136 00000100:00100000:3.0:1767319975.644544:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307136 00000100:00100000:3.0:1767319975.644598:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307264 00000100:00100000:3.0:1767319975.644601:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307264 00000100:00100000:3.0:1767319975.644621:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2414 00000100:00100000:3.0:1767319975.644629:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb2050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985307136:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319975.644655:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319975.644688:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb2050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985307136:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (669us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319975.644694:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2414 00000100:00080000:2.0:1767319975.644812:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319975.644817:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319975.644835:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2415 00000100:00100000:2.0:1767319975.644843:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985307264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319975.644859:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319975.644892:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985307264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (400us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319975.644898:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2415 00000100:00100000:3.0:1767319975.644914:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985307264:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319975.646139:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985307136:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319975.775856:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319976.109074:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319976.109128:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319976.109220:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920591104 00000100:00100000:1.0:1767319976.109232:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920591104 00000100:00100000:2.0:1767319976.109242:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920590976 00000100:00100000:2.0:1767319976.109246:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920590976 00000100:00100000:2.0:1767319976.109334:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4943 00000100:00100000:2.0:1767319976.109345:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920591104:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319976.109424:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920591104:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (360us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319976.109432:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4943 00000100:00100000:2.0:1767319976.109449:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5529 00000100:00100000:2.0:1767319976.109454:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920590976:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319976.109477:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319976.109502:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920590976:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (383us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319976.109508:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5529 00000100:00080000:3.0:1767319976.109864:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319976.156575:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319976.156586:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319976.156589:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319976.156593:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319976.156598:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319976.156606:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5219 00000100:00080000:0.0:1767319976.156610:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319976.156616:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319976.156649:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1c00 x1853163985307392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319976.156674:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319976.156682:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319976.156684:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319976.156687:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5219 00000100:00080000:0.0:1767319976.156689:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319976.156692:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319976.156705:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0000 x1853163985307520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319976.156735:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319976.156739:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319976.156741:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319976.156744:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5219 00000100:00080000:0.0:1767319976.156746:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319976.156750:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319976.156763:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1f80 x1853163985307776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319976.156790:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319976.156792:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319976.156795:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319976.156797:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5219 00000100:00080000:0.0:1767319976.156800:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319976.156803:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319976.156814:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfad80 x1853163985308032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319976.156861:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985308160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319976.156912:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985307648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319976.156930:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319976.157002:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985307904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319976.157162:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985307392:0@lo:38:kworker.0 00000100:00100000:3.0:1767319976.162808:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319976.162940:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308160 00000100:00100000:3.0:1767319976.162952:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308160 00000100:00100000:3.0:1767319976.163005:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3229 00000100:00100000:3.0:1767319976.163015:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985308160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319976.163097:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985308160:12345-0@lo:400:kworker.0 Request processed in 83us (6167us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319976.163104:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3229 00000100:00100000:3.0:1767319976.163146:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307648 00000100:00100000:3.0:1767319976.163151:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307648 00000100:00100000:3.0:1767319976.163207:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4944 00000100:00100000:3.0:1767319976.163213:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985307648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319976.163251:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985307648:12345-0@lo:400:kworker.0 Request processed in 38us (451us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319976.163257:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4944 00000100:00080000:3.0:1767319976.163272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319976.166593:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319976.166688:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985308288:0@lo:400:kworker.0 00000100:00100000:1.0:1767319976.166729:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319976.166758:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308288 00000100:00100000:1.0:1767319976.166775:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308288 00000100:00100000:1.0:1767319976.166816:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3230 00000100:00100000:1.0:1767319976.166823:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985308288:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319976.166869:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985308288:12345-0@lo:400:kworker.0 Request processed in 46us (139us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319976.166876:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3230 00000100:00100000:1.0:1767319976.166915:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985308288:0@lo:400:kworker.0 00000100:00100000:1.0:1767319976.166945:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307904 00000100:00100000:1.0:1767319976.166948:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307904 00000100:00100000:1.0:1767319976.166990:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4945 00000100:00100000:1.0:1767319976.166996:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985307904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319976.167031:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985307904:12345-0@lo:400:kworker.0 Request processed in 35us (444us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319976.167036:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4945 00000100:00100000:1.0:1767319976.167060:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985307904:0@lo:400:kworker.0 00000100:00080000:1.0:1767319976.167088:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319976.167241:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985307648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319976.167857:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985308160:0@lo:400:kworker.0 00000100:00100000:2.0:1767319976.170639:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319976.170952:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985307520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319976.170994:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319976.171032:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985307776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319976.171053:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319976.171120:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307520 00000100:00100000:3.0:1767319976.171129:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307520 00000100:00100000:3.0:1767319976.171194:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1873 00000100:00100000:3.0:1767319976.171202:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985307520:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319976.171224:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319976.171269:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985307520:12345-0@lo:38:kworker.0 Request processed in 66us (274us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319976.171289:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1873 00000100:00100000:2.0:1767319976.176968:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985308032:0@lo:38:kworker.0 00000100:00100000:2.0:1767319976.177034:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319976.178645:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307392 00000100:00100000:1.0:1767319976.178658:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307392 00000100:00100000:1.0:1767319976.178728:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985307776 00000100:00100000:1.0:1767319976.178731:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985307776 00000100:00100000:1.0:1767319976.178751:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5530 00000100:00100000:1.0:1767319976.178758:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985307392:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319976.178791:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319976.178865:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985307392:12345-0@lo:38:kworker.0 Request processed in 106us (8232us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319976.178872:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5530 00000100:00100000:1.0:1767319976.178887:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5531 00000100:00100000:1.0:1767319976.178891:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985307776:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319976.178901:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319976.178924:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985307776:12345-0@lo:38:kworker.0 Request processed in 31us (7870us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319976.178930:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5531 00000100:00080000:2.0:1767319976.182507:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319976.182522:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319976.182527:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985307392:0@lo:38:kworker.0 00000100:00080000:2.0:1767319976.182541:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319976.182544:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319976.182547:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985307520:0@lo:38:kworker.0 00000100:00080000:2.0:1767319976.182557:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319976.182560:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319976.182563:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985307776:0@lo:38:kworker.0 00000100:00100000:1.0:1767319976.182839:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308032 00000100:00100000:1.0:1767319976.182848:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308032 00000100:00100000:1.0:1767319976.182894:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5532 00000100:00100000:1.0:1767319976.182902:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985308032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319976.182922:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319976.182991:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985308032:12345-0@lo:38:kworker.0 Request processed in 89us (5957us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319976.182997:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5532 00000100:00080000:2.0:1767319976.183103:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319976.183108:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319976.183111:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985308032:0@lo:38:kworker.0 00010000:00010000:0.0:1767319976.795496:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319977.826686:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319978.843459:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319979.867768:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319980.763861:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985308544:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319980.763930:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319980.764006:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319980.764012:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985308416:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319980.764036:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319980.764110:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308544 00000100:00100000:1.0:1767319980.764120:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308544 00000100:00100000:1.0:1767319980.764211:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308416 00000100:00100000:1.0:1767319980.764215:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308416 00000100:00100000:1.0:1767319980.764239:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2416 00000100:00100000:1.0:1767319980.764249:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985308544:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319980.764261:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2417 00000100:00100000:2.0:1767319980.764275:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985308416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319980.764281:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:2.0:1767319980.764295:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767319980.764340:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985308544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (409us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319980.764343:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985308416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319980.764354:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2416 00000100:00080000:3.0:1767319980.764392:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319980.764396:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319980.764563:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985308416:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319980.764577:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985308544:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319980.772075:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2417 00010000:00010000:2.0:1767319980.892533:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319981.242829:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319981.242875:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319981.243283:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920591360 00000100:00100000:0.0:1767319981.243294:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920591360 00000100:00100000:0.0:1767319981.243376:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920591232 00000100:00100000:0.0:1767319981.243380:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920591232 00000100:00100000:0.0:1767319981.243399:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4946 00000100:00100000:0.0:1767319981.243410:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920591360:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319981.243488:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920591360:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (670us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319981.243495:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4946 00000100:00100000:0.0:1767319981.243514:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5533 00000100:00100000:0.0:1767319981.243520:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920591232:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319981.243539:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319981.243567:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920591232:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (691us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319981.243573:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5533 00000100:00080000:1.0:1767319981.244254:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319981.275326:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319981.275336:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319981.275340:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319981.275344:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319981.275348:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319981.275356:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5224 00000100:00080000:0.0:1767319981.275361:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319981.275367:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319981.275404:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9880 x1853163985308672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319981.275436:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319981.275440:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319981.275441:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319981.275445:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5224 00000100:00080000:0.0:1767319981.275447:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319981.275451:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319981.275463:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfad80 x1853163985308800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319981.275494:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319981.275497:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319981.275499:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319981.275502:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5224 00000100:00080000:0.0:1767319981.275505:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319981.275507:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319981.275521:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb800 x1853163985309056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319981.275538:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319981.275541:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319981.275543:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319981.275546:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5224 00000100:00080000:0.0:1767319981.275548:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319981.275551:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319981.275561:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bce00 x1853163985309312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319981.275660:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985308672:0@lo:38:kworker.0 00000100:00100000:2.0:1767319981.275736:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319981.275798:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985308800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319981.275825:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319981.275850:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985309056:0@lo:38:kworker.0 00000100:00100000:2.0:1767319981.275869:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319981.275883:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985309312:0@lo:38:kworker.0 00000100:00100000:2.0:1767319981.275902:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319981.275940:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308800 00000100:00100000:2.0:1767319981.275947:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308800 00000100:00100000:2.0:1767319981.275979:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1874 00000100:00100000:2.0:1767319981.275986:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985308800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319981.276005:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319981.276041:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985308800:12345-0@lo:38:kworker.0 Request processed in 54us (214us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319981.276047:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1874 00000100:00080000:2.0:1767319981.276084:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319981.276088:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319981.276092:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985308800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319981.276127:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308672 00000100:00100000:2.0:1767319981.276130:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308672 00000100:00100000:2.0:1767319981.276150:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309056 00000100:00100000:2.0:1767319981.276153:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309056 00000100:00100000:2.0:1767319981.276184:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309312 00000100:00100000:2.0:1767319981.276187:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309312 00000100:00100000:2.0:1767319981.276202:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5534 00000100:00100000:2.0:1767319981.276206:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985308672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319981.276216:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319981.276244:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985308672:12345-0@lo:38:kworker.0 Request processed in 36us (508us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319981.276250:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5534 00000100:00100000:2.0:1767319981.276261:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5535 00000100:00100000:2.0:1767319981.276265:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985309056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319981.276274:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319981.276294:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985309056:12345-0@lo:38:kworker.0 Request processed in 28us (424us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319981.276300:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5535 00000100:00100000:2.0:1767319981.276310:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5536 00000100:00100000:2.0:1767319981.276313:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985309312:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319981.276322:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319981.276342:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985309312:12345-0@lo:38:kworker.0 Request processed in 27us (440us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319981.276347:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5536 00000100:00080000:2.0:1767319981.276378:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319981.276382:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319981.276385:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985308672:0@lo:38:kworker.0 00000100:00080000:2.0:1767319981.276392:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319981.276395:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319981.276398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985309056:0@lo:38:kworker.0 00000100:00080000:2.0:1767319981.276405:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319981.276407:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319981.276410:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985309312:0@lo:38:kworker.0 00000100:00100000:3.0:1767319981.276608:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985308928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319981.276642:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319981.276682:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319981.276689:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985309184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319981.276710:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319981.276739:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985308928 00000100:00100000:3.0:1767319981.276743:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985308928 00000100:00100000:3.0:1767319981.276778:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309184 00000100:00100000:3.0:1767319981.276782:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309184 00000100:00100000:3.0:1767319981.276801:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4947 00000100:00100000:3.0:1767319981.276806:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985308928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319981.276847:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985308928:12345-0@lo:400:kworker.0 Request processed in 41us (204us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319981.276853:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4947 00000100:00100000:3.0:1767319981.276866:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4948 00000100:00100000:3.0:1767319981.276871:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985309184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319981.276896:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985309184:12345-0@lo:400:kworker.0 Request processed in 25us (185us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319981.276902:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4948 00000100:00080000:3.0:1767319981.276921:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319981.276940:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985309184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319981.276949:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985308928:0@lo:400:kworker.0 00010000:00010000:1.0:1767319981.916436:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319982.943695:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319983.966059:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319984.987799:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319985.883547:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985309568:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319985.883625:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319985.883704:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319985.883711:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985309440:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319985.883733:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319985.883814:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309568 00000100:00100000:2.0:1767319985.883823:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309568 00000100:00100000:2.0:1767319985.883877:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309440 00000100:00100000:2.0:1767319985.883881:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309440 00000100:00100000:2.0:1767319985.883899:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2418 00000100:00100000:2.0:1767319985.883907:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985309568:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319985.883934:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319985.883973:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985309568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (347us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319985.883980:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2418 00000100:00100000:2.0:1767319985.883996:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2419 00000100:00100000:2.0:1767319985.884002:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985309440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319985.884015:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319985.884037:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985309440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 35us (303us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319985.884042:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2419 00000100:00080000:0.0:1767319985.884094:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319985.884098:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319985.884130:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985309440:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319985.884141:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985309568:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319986.011352:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319986.349121:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319986.349345:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319986.350090:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920591488 00000100:00100000:2.0:1767319986.350101:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920591488 00000100:00100000:2.0:1767319986.350143:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920591616 00000100:00100000:2.0:1767319986.350146:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920591616 00000100:00100000:2.0:1767319986.350213:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4949 00000100:00100000:2.0:1767319986.350225:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920591616:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319986.350298:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920591616:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (952us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319986.350306:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4949 00000100:00100000:2.0:1767319986.350326:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5537 00000100:00100000:2.0:1767319986.350330:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920591488:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319986.350350:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319986.350374:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920591488:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (1265us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319986.350380:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5537 00000100:00080000:0.0:1767319986.351119:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319986.395534:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319986.395545:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319986.395547:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319986.395552:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319986.395557:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319986.395564:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5229 00000100:00080000:1.0:1767319986.395568:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319986.395574:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319986.395624:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb100 x1853163985309696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319986.395650:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319986.395654:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319986.395656:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319986.395660:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5229 00000100:00080000:1.0:1767319986.395662:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319986.395665:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319986.395676:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8000 x1853163985309824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319986.395714:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319986.395717:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319986.395719:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319986.395722:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5229 00000100:00080000:1.0:1767319986.395725:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319986.395728:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319986.395741:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9500 x1853163985310080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319986.395836:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985309696:0@lo:38:kworker.0 00000100:00100000:0.0:1767319986.395912:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319986.395968:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985309824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319986.395992:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319986.396013:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985310080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319986.396034:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319986.396085:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309696 00000100:00100000:0.0:1767319986.396093:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309696 00000100:00100000:0.0:1767319986.396120:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985310080 00000100:00100000:0.0:1767319986.396122:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985310080 00000100:00100000:0.0:1767319986.396149:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5538 00000100:00100000:0.0:1767319986.396156:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985309696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319986.396202:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319986.396244:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985309696:12345-0@lo:38:kworker.0 Request processed in 86us (330us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319986.396251:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5538 00000100:00100000:0.0:1767319986.396268:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5539 00000100:00100000:0.0:1767319986.396272:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985310080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319986.396281:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319986.396303:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985310080:12345-0@lo:38:kworker.0 Request processed in 30us (269us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319986.396309:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5539 00000100:00080000:0.0:1767319986.396341:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319986.396346:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319986.396349:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985309696:0@lo:38:kworker.0 00000100:00080000:0.0:1767319986.396357:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319986.396360:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319986.396363:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985310080:0@lo:38:kworker.0 00000100:00100000:3.0:1767319986.396545:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309824 00000100:00100000:3.0:1767319986.396549:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309824 00000100:00100000:3.0:1767319986.396589:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1875 00000100:00100000:3.0:1767319986.396593:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985309824:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319986.396603:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319986.396638:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985309824:12345-0@lo:38:kworker.0 Request processed in 43us (645us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319986.396644:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1875 00000100:00080000:3.0:1767319986.396669:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319986.396673:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319986.396675:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985309824:0@lo:38:kworker.0 00000100:00100000:3.0:1767319986.396693:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985309952:0@lo:400:kworker.0 00000100:00100000:3.0:1767319986.396717:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319986.396746:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985309952 00000100:00100000:3.0:1767319986.396749:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985309952 00000100:00100000:3.0:1767319986.396798:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4950 00000100:00100000:3.0:1767319986.396804:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985309952:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319986.396844:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985309952:12345-0@lo:400:kworker.0 Request processed in 40us (126us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319986.396851:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4950 00000100:00080000:3.0:1767319986.396879:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319986.396901:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985309952:0@lo:400:kworker.0 00000100:00080000:1.0:1767319986.397190:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319986.397195:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319986.397198:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319986.397202:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5229 00000100:00080000:1.0:1767319986.397205:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319986.397209:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767319986.397230:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bda680 x1853163985310336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319986.397297:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985310208:0@lo:400:kworker.0 00000100:00100000:2.0:1767319986.397327:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319986.397404:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985310208 00000100:00100000:1.0:1767319986.397409:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985310208 00000100:00100000:1.0:1767319986.397448:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4951 00000100:00100000:1.0:1767319986.397455:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985310208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319986.397491:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985310208:12345-0@lo:400:kworker.0 Request processed in 37us (163us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319986.397497:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4951 00000100:00080000:2.0:1767319986.397680:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319986.397704:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985310208:0@lo:400:kworker.0 00000100:00100000:3.0:1767319986.398058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985310336:0@lo:38:kworker.0 00000100:00100000:3.0:1767319986.398089:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319986.398191:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985310336 00000100:00100000:2.0:1767319986.398196:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985310336 00000100:00100000:2.0:1767319986.398227:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5540 00000100:00100000:2.0:1767319986.398231:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985310336:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319986.398247:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319986.398285:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985310336:12345-0@lo:38:kworker.0 Request processed in 51us (195us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319986.398291:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5540 00000100:00080000:3.0:1767319986.398351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319986.398356:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319986.398359:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985310336:0@lo:38:kworker.0 00010000:00010000:2.0:1767319987.043716:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319988.059394:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319989.083385:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319990.107740:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319990.446530:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767319990.446833:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592000 00000100:00100000:2.0:1767319990.446851:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592000 00000100:00100000:2.0:1767319990.446958:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4952 00000100:00100000:2.0:1767319990.446972:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920592000:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767319990.447068:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920592000:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (552us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319990.447077:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4952 00000100:00080000:3.0:1767319990.453274:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:2.0:1767319990.453514:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920591744 offset 0 length 520: 4 00000100:00100000:2.0:1767319990.453559:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319990.453657:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920591872 00000100:00100000:0.0:1767319990.453667:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920591872 00000100:00100000:0.0:1767319990.453709:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5541 00000100:00100000:0.0:1767319990.453715:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920591872:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767319990.453735:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319990.453807:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920591872:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (248us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319990.453813:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5541 00000100:00100000:3.0:1767319990.491412:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319965/real 1767319965] req@ffff8ce1af18f100 x1853163985304704/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319990 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319990.491442:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1af18f100 x1853163985304704/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319990 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319990.491454:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.491485:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319990.491488:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985304704:0@lo:250:kworker.0 00000100:00080000:0.0:1767319990.491587:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319990.491592:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319990.491596:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319990.491600:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5214 00000100:00080000:0.0:1767319990.491604:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319990.491611:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319990.491648:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5500 x1853163985310464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319990.491671:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319990.491674:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319990.491676:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319990.491678:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5234 00000100:00080000:0.0:1767319990.491680:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319990.491683:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319990.491694:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5880 x1853163985310592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319990.491704:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319990.491706:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319990.491707:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319990.491708:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5234 00000100:00080000:0.0:1767319990.491710:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319990.491711:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319990.491719:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9db480 x1853163985310720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319990.491747:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319990.491749:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319990.491751:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319990.491753:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5234 00000100:00100000:1.0:1767319990.491856:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985310848:0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.491911:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319990.491995:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985310848 00000100:00100000:2.0:1767319990.492002:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985310848 00000100:00100000:2.0:1767319990.492051:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4953 00000100:00100000:2.0:1767319990.492057:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985310848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319990.492103:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985310848:12345-0@lo:400:kworker.0 Request processed in 46us (191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319990.492110:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4953 00000100:00080000:1.0:1767319990.492190:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319990.492216:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985310848:0@lo:400:kworker.0 00000100:00100000:3.0:1767319990.492273:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985310464:0@lo:250:kworker.0 00000400:00000100:3.0:1767319990.492298:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985310464 offset 0 length 520: 4 00000100:00100000:3.0:1767319990.492317:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985310592:0@lo:38:kworker.0 00000100:00100000:3.0:1767319990.492337:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319990.492359:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9db480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985310720:0@lo:38:kworker.0 00000100:00100000:3.0:1767319990.492382:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319990.492450:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985310720 00000100:00100000:1.0:1767319990.492455:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985310720 00000100:00100000:1.0:1767319990.492485:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1876 00000100:00100000:1.0:1767319990.492490:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985310720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319990.492518:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319990.492550:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985310720:12345-0@lo:38:kworker.0 Request processed in 59us (167us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319990.492555:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1876 00000100:00100000:2.0:1767319990.492607:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985310592 00000100:00100000:2.0:1767319990.492611:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985310592 00000100:00100000:2.0:1767319990.492639:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5542 00000100:00100000:2.0:1767319990.492644:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985310592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319990.492654:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319990.492675:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985310592:12345-0@lo:38:kworker.0 Request processed in 31us (338us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319990.492680:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5542 00000100:00080000:3.0:1767319990.492752:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.492764:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.492767:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985310592:0@lo:38:kworker.0 00000100:00080000:3.0:1767319990.492772:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.492775:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.492776:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9db480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985310720:0@lo:38:kworker.0 00000100:00080000:0.0:1767319990.492881:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319990.492886:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319990.492901:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9880 x1853163985310976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319990.492930:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319990.492932:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319990.492933:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319990.492936:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5234 00000100:00080000:0.0:1767319990.492938:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319990.492940:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767319990.492946:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9d9180 x1853163985311232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319990.493029:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319990.493036:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985311104:0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.493056:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319990.493124:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985311488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319990.493153:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319990.493196:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319990.493201:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985311360:0@lo:400:kworker.0 00000100:00100000:2.0:1767319990.493217:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319990.493282:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985311488 00000100:00100000:1.0:1767319990.493287:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985311488 00000100:00100000:1.0:1767319990.493329:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985311360 00000100:00100000:1.0:1767319990.493332:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985311360 00000100:00100000:1.0:1767319990.493351:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3231 00000100:00100000:1.0:1767319990.493357:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985311488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.493394:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985311488:12345-0@lo:400:kworker.0 Request processed in 37us (241us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319990.493401:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3231 00000100:00100000:1.0:1767319990.493413:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3232 00000100:00100000:1.0:1767319990.493419:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985311360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.493441:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985311360:12345-0@lo:400:kworker.0 Request processed in 23us (222us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319990.493445:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3232 00000100:00100000:1.0:1767319990.493471:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985311360:0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.493479:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985311488:0@lo:400:kworker.0 00000100:00080000:2.0:1767319990.493542:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319990.493546:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319990.493583:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985311104 00000100:00100000:0.0:1767319990.493586:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985311104 00000100:00100000:0.0:1767319990.493632:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4954 00000100:00100000:0.0:1767319990.493638:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985311104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319990.493670:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985311104:12345-0@lo:400:kworker.0 Request processed in 33us (613us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319990.493674:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4954 00000100:00100000:1.0:1767319990.493725:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985311104:0@lo:400:kworker.0 00000100:00080000:2.0:1767319990.493937:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319990.494039:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985310976:0@lo:38:kworker.0 00000100:00100000:3.0:1767319990.494065:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319990.494090:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985311232:0@lo:38:kworker.0 00000100:00100000:3.0:1767319990.494110:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319990.494197:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985310976 00000100:00100000:1.0:1767319990.494202:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985310976 00000100:00100000:1.0:1767319990.494221:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985311232 00000100:00100000:1.0:1767319990.494223:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985311232 00000100:00100000:1.0:1767319990.494240:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5543 00000100:00100000:1.0:1767319990.494243:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985310976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319990.494256:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319990.494285:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985310976:12345-0@lo:38:kworker.0 Request processed in 40us (219us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319990.494291:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5543 00000100:00100000:1.0:1767319990.494302:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5544 00000100:00100000:1.0:1767319990.494306:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985311232:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319990.494315:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319990.494336:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985311232:12345-0@lo:38:kworker.0 Request processed in 29us (225us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319990.494342:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5544 00000100:00080000:3.0:1767319990.494513:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.494518:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.494522:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985310976:0@lo:38:kworker.0 00000100:00080000:3.0:1767319990.494532:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.494535:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.494538:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985311232:0@lo:38:kworker.0 00000100:00100000:0.0:1767319991.003553:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319991.003576:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985311744:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319991.003648:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319991.003689:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985311616:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319991.003710:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319991.003906:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985311744 00000100:00100000:2.0:1767319991.003914:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985311744 00000100:00100000:2.0:1767319991.003962:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985311616 00000100:00100000:2.0:1767319991.003965:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985311616 00000100:00100000:2.0:1767319991.003995:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2420 00000100:00100000:2.0:1767319991.004004:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985311744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319991.004034:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319991.004078:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985311744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (429us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319991.004086:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2420 00000100:00100000:2.0:1767319991.004103:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2421 00000100:00100000:2.0:1767319991.004109:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985311616:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319991.004124:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319991.004149:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985311616:12345-0@lo:13:osp-pre-1-1.0 Request processed in 40us (438us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319991.004154:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2421 00000100:00100000:0.0:1767319991.004255:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985311744:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319991.004266:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985311616:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319991.131429:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319992.155411:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319993.179406:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319994.203402:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319995.227596:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319995.562069:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767319995.562121:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767319995.562209:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592256 00000100:00100000:2.0:1767319995.562209:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592128 00000100:00100000:0.0:1767319995.562220:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592256 00000100:00100000:2.0:1767319995.562221:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592128 00000100:00100000:2.0:1767319995.562275:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5545 00000100:00100000:2.0:1767319995.562283:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920592128:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767319995.562307:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319995.562317:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319995.562392:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920592128:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 107us (332us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319995.564664:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4955 00000100:00100000:0.0:1767319995.564688:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920592256:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767319995.564772:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920592256:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 98us (2650us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319995.564780:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4955 00000100:00100000:2.0:1767319995.565570:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5545 00000100:00080000:3.0:1767319995.612015:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319995.612024:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319995.612027:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319995.612031:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319995.612036:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319995.612045:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5238 00000100:00080000:3.0:1767319995.612049:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319995.612056:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319995.612097:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5880 x1853163985311872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319995.612197:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319995.612201:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319995.612203:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319995.612206:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5238 00000100:00080000:3.0:1767319995.612209:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319995.612212:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319995.612224:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579500 x1853163985312000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319995.612261:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319995.612264:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319995.612265:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319995.612268:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5238 00000100:00080000:3.0:1767319995.612271:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319995.612274:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319995.612285:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7c000 x1853163985312256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319995.612300:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319995.612303:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319995.612305:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319995.612308:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5238 00000100:00080000:3.0:1767319995.612310:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319995.612313:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767319995.612324:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631500 x1853163985312512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319995.612337:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985312128:0@lo:400:kworker.0 00000100:00100000:3.0:1767319995.612374:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985312768:0@lo:400:kworker.0 00000100:00100000:3.0:1767319995.612428:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319995.612431:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319995.612509:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985311872:0@lo:38:kworker.0 00000100:00100000:0.0:1767319995.612539:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319995.612571:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312128 00000100:00100000:0.0:1767319995.612578:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312128 00000100:00100000:0.0:1767319995.612627:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985311872 00000100:00100000:0.0:1767319995.612630:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985311872 00000100:00100000:0.0:1767319995.612647:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4956 00000100:00100000:0.0:1767319995.612655:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985312128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319995.612701:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985312128:12345-0@lo:400:kworker.0 Request processed in 47us (277us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319995.612708:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4956 00000100:00100000:0.0:1767319995.612724:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5546 00000100:00100000:0.0:1767319995.612728:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985311872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319995.612743:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319995.612768:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985311872:12345-0@lo:38:kworker.0 Request processed in 39us (228us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319995.612773:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5546 00000100:00100000:0.0:1767319995.612804:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312768 00000100:00100000:0.0:1767319995.612807:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312768 00000100:00100000:0.0:1767319995.612829:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3233 00000100:00100000:0.0:1767319995.612834:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985312768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.612925:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985312384:0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.612949:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319995.612972:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312384 00000100:00100000:1.0:1767319995.612975:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312384 00000100:00100000:1.0:1767319995.613002:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4957 00000100:00100000:1.0:1767319995.613008:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985312384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.613040:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985312384:12345-0@lo:400:kworker.0 Request processed in 32us (89us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319995.613045:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4957 00000100:00080000:1.0:1767319995.613073:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319995.613076:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319995.613087:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985312640:0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.613107:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.613226:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312640 00000100:00100000:3.0:1767319995.613229:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312640 00000100:00100000:3.0:1767319995.613251:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3234 00000100:00100000:3.0:1767319995.613256:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985312640:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319995.613284:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985312640:12345-0@lo:400:kworker.0 Request processed in 27us (175us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319995.613290:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3234 00000100:00080000:3.0:1767319995.613363:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.613368:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.613371:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985311872:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.613378:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985312000:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.613402:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.613435:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985312256:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.613455:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.613469:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985312512:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.613487:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.613523:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312000 00000100:00100000:3.0:1767319995.613527:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312000 00000100:00100000:3.0:1767319995.613547:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1877 00000100:00100000:3.0:1767319995.613550:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985312000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319995.613562:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319995.613591:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985312000:12345-0@lo:38:kworker.0 Request processed in 38us (188us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319995.613597:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1877 00000100:00100000:3.0:1767319995.613611:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312256 00000100:00100000:3.0:1767319995.613614:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312256 00000100:00100000:3.0:1767319995.613628:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312512 00000100:00100000:3.0:1767319995.613630:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312512 00000100:00100000:3.0:1767319995.613641:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5547 00000100:00100000:3.0:1767319995.613644:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985312256:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319995.613653:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319995.613672:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985312256:12345-0@lo:38:kworker.0 Request processed in 27us (216us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319995.613677:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5547 00000100:00100000:3.0:1767319995.613685:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5548 00000100:00100000:3.0:1767319995.613688:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985312512:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319995.613695:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319995.613709:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985312512:12345-0@lo:38:kworker.0 Request processed in 20us (222us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319995.613714:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5548 00000100:00080000:3.0:1767319995.613733:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.613736:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.613739:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985312000:0@lo:38:kworker.0 00000100:00080000:3.0:1767319995.613745:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.613748:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.613750:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985312256:0@lo:38:kworker.0 00000100:00080000:3.0:1767319995.613755:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.613758:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.613760:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985312512:0@lo:38:kworker.0 00000100:00080000:3.0:1767319995.613786:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319995.613800:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985312384:0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.613904:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985312640:0@lo:400:kworker.0 00000100:00100000:0.0:1767319995.613998:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985312768:12345-0@lo:400:kworker.0 Request processed in 1162us (1572us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319995.614006:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3233 00000100:00100000:3.0:1767319995.616354:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985312768:0@lo:400:kworker.0 00000100:00100000:3.0:1767319995.620041:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985312128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319996.123820:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319996.123844:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985312896:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319996.123912:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319996.123997:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985312896 00000100:00100000:2.0:1767319996.124004:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985312896 00000100:00100000:2.0:1767319996.124390:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2422 00000100:00100000:2.0:1767319996.124409:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985312896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319996.124439:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767319996.125130:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319996.125200:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985312896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 800us (1286us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319996.125209:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2422 00000100:00100000:3.0:1767319996.125304:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985313024:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319996.125334:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319996.125443:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985312896:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319996.125600:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313024 00000100:00100000:2.0:1767319996.125606:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313024 00000100:00100000:2.0:1767319996.125646:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2423 00000100:00100000:2.0:1767319996.125653:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985313024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319996.125671:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767319996.125715:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985313024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319996.125721:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2423 00000100:00100000:0.0:1767319996.125961:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985313024:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319996.256192:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319997.275409:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319998.300235:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319999.323416:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320000.347731:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320000.691496:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320000.691540:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320000.693800:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592384 00000100:00100000:3.0:1767320000.693815:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592384 00000100:00100000:3.0:1767320000.693972:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592512 00000100:00100000:3.0:1767320000.693976:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592512 00000100:00100000:3.0:1767320000.694035:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4958 00000100:00100000:3.0:1767320000.694047:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920592512:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320000.694126:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920592512:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (2585us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320000.694133:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4958 00000100:00100000:3.0:1767320000.694150:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5549 00000100:00100000:3.0:1767320000.694155:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920592384:12345-192.168.204.26@tcp:38:kworker.0 00000100:00080000:0.0:1767320000.694250:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:3.0:1767320000.694634:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320000.694745:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920592384:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 586us (3257us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320000.694753:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5549 00000100:00080000:1.0:1767320000.732009:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.732019:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320000.732022:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.732027:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.732032:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.732040:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.732045:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.732052:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320000.732093:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8000 x1853163985313152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320000.732126:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.732130:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.732132:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.732135:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.732138:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.732141:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320000.732154:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d500 x1853163985313280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320000.732269:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985313152:0@lo:38:kworker.0 00000100:00100000:3.0:1767320000.732376:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320000.732484:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313152 00000100:00100000:2.0:1767320000.732495:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313152 00000100:00100000:2.0:1767320000.732552:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5550 00000100:00100000:2.0:1767320000.732559:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985313152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320000.732581:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320000.732665:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985313152:12345-0@lo:38:kworker.0 Request processed in 105us (289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320000.732673:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5550 00000100:00080000:0.0:1767320000.732780:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320000.732785:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320000.732789:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985313152:0@lo:38:kworker.0 00000100:00100000:0.0:1767320000.732875:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985313280:0@lo:38:kworker.0 00000100:00100000:0.0:1767320000.732973:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320000.733052:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.733057:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.733059:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.733063:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.733065:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.733069:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320000.733082:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985313536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320000.733099:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.733103:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.733105:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.733110:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.733112:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.733117:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320000.733128:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a300 x1853163985313792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320000.733141:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985313664:0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.733223:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320000.733235:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985313920:0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.733263:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320000.733275:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985313408:0@lo:400:kworker.0 00000100:00100000:1.0:1767320000.733287:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320000.733346:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320000.733404:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313664 00000100:00100000:3.0:1767320000.733412:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313664 00000100:00100000:3.0:1767320000.733471:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313408 00000100:00100000:3.0:1767320000.733475:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313408 00000100:00100000:3.0:1767320000.733501:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4959 00000100:00100000:3.0:1767320000.733508:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985313664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.733563:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985313664:12345-0@lo:400:kworker.0 Request processed in 55us (341us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320000.733570:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4959 00000100:00100000:3.0:1767320000.733586:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4960 00000100:00100000:3.0:1767320000.733592:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985313408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.733637:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985313408:12345-0@lo:400:kworker.0 Request processed in 45us (291us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320000.733643:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4960 00000100:00100000:3.0:1767320000.733697:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985313408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.733719:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985313664:0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.733801:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313920 00000100:00100000:3.0:1767320000.733806:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313920 00000100:00100000:3.0:1767320000.733841:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3235 00000100:00100000:3.0:1767320000.733848:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985313920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.733876:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985313920:12345-0@lo:400:kworker.0 Request processed in 31us (593us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320000.733881:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3235 00000100:00080000:3.0:1767320000.733900:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320000.733905:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320000.733923:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985314048:0@lo:400:kworker.0 00000100:00100000:1.0:1767320000.734083:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985313920:0@lo:400:kworker.0 00000100:00100000:0.0:1767320000.734286:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985313536:0@lo:38:kworker.0 00000100:00100000:0.0:1767320000.734340:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320000.734371:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985313792:0@lo:38:kworker.0 00000100:00100000:0.0:1767320000.734390:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320000.734424:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313536 00000100:00100000:0.0:1767320000.734430:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313536 00000100:00100000:0.0:1767320000.734458:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313792 00000100:00100000:0.0:1767320000.734461:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313792 00000100:00100000:0.0:1767320000.734477:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5551 00000100:00100000:0.0:1767320000.734484:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985313536:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320000.734503:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320000.734537:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985313536:12345-0@lo:38:kworker.0 Request processed in 53us (197us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320000.734544:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5551 00000100:00100000:0.0:1767320000.734557:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5552 00000100:00100000:0.0:1767320000.734561:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9db480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985313792:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320000.734570:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320000.734590:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9db480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985313792:12345-0@lo:38:kworker.0 Request processed in 27us (199us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320000.734596:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5552 00000100:00080000:0.0:1767320000.734623:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320000.734628:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320000.734631:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985313536:0@lo:38:kworker.0 00000100:00080000:0.0:1767320000.734639:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320000.734642:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320000.734645:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985313792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320000.735272:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985313280 00000100:00100000:2.0:1767320000.735278:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985313280 00000100:00100000:3.0:1767320000.739365:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320000.740216:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314048 00000100:00100000:3.0:1767320000.740230:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314048 00000100:00100000:3.0:1767320000.740307:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3236 00000100:00080000:1.0:1767320000.741429:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320000.741559:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985314048:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.742850:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985314048:12345-0@lo:400:kworker.0 Request processed in 179us (2366us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320000.742872:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3236 00000100:00100000:1.0:1767320000.743565:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985314048:0@lo:400:kworker.0 00000100:00100000:2.0:1767320000.743757:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1878 00000100:00100000:2.0:1767320000.743771:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985313280:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320000.743797:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320000.743911:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985313280:12345-0@lo:38:kworker.0 Request processed in 130us (10926us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320000.743924:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1878 00000100:00080000:1.0:1767320000.743953:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320000.743961:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320000.743965:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985313280:0@lo:38:kworker.0 00000100:00100000:0.0:1767320001.243813:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320001.243843:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985314176:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320001.243952:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320001.244073:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314176 00000100:00100000:1.0:1767320001.244084:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314176 00000100:00100000:1.0:1767320001.244160:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2424 00000100:00100000:1.0:1767320001.244207:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9d9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985314176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320001.244247:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320001.244320:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9d9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985314176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 116us (367us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320001.244328:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2424 00000100:00100000:0.0:1767320001.244584:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985314176:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320001.245889:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985314304:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320001.245966:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320001.246274:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314304 00000100:00100000:1.0:1767320001.246284:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314304 00000100:00100000:1.0:1767320001.246579:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2425 00000100:00100000:1.0:1767320001.246591:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985314304:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320001.246631:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320001.246697:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985314304:12345-0@lo:13:osp-pre-0-1.0 Request processed in 107us (730us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320001.246706:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2425 00000100:00100000:0.0:1767320001.246793:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985314304:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320001.371413:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320002.399275:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320003.419800:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320004.444788:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320005.468348:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320005.806930:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320005.806987:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320005.807069:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592768 00000100:00100000:3.0:1767320005.807082:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592768 00000100:00100000:3.0:1767320005.807198:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592640 00000100:00100000:3.0:1767320005.807202:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592640 00000100:00100000:3.0:1767320005.807259:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4961 00000100:00100000:3.0:1767320005.807273:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920592768:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320005.807377:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920592768:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 108us (460us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320005.807385:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4961 00000100:00100000:3.0:1767320005.807407:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5553 00000100:00100000:3.0:1767320005.807411:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920592640:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320005.807434:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320005.807941:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920592640:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 525us (951us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320005.807949:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5553 00000100:00080000:1.0:1767320005.808116:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320005.851736:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.851750:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320005.851755:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.851760:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.851765:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.851777:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.851782:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.851792:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320005.851846:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593bb80 x1853163985314432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320005.851880:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.851884:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.851886:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.851891:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.851894:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.851898:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320005.851912:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939f80 x1853163985314560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320005.851961:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.851965:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.851967:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.851971:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.851974:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.851977:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320005.851993:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593aa00 x1853163985314816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320005.852011:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.852014:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.852016:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.852020:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.852022:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.852026:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320005.852037:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985315072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320005.852123:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985315200:0@lo:400:kworker.0 00000100:00100000:1.0:1767320005.852245:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320005.852326:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320005.852332:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985315328:0@lo:400:kworker.0 00000100:00100000:1.0:1767320005.852356:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320005.853672:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985314688:0@lo:400:kworker.0 00000100:00100000:2.0:1767320005.853740:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320005.853804:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320005.853810:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985314944:0@lo:400:kworker.0 00000100:00100000:2.0:1767320005.853839:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320005.853963:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985314432:0@lo:38:kworker.0 00000100:00100000:0.0:1767320005.854002:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320005.854020:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985314560:0@lo:38:kworker.0 00000100:00100000:0.0:1767320005.854044:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320005.854072:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985314816:0@lo:38:kworker.0 00000100:00100000:0.0:1767320005.854097:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320005.854117:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985315072:0@lo:38:kworker.0 00000100:00100000:0.0:1767320005.854137:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320005.854199:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314560 00000100:00100000:0.0:1767320005.854209:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314560 00000100:00100000:0.0:1767320005.854265:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1879 00000100:00100000:0.0:1767320005.854274:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985314560:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320005.854299:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320005.854348:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985314560:12345-0@lo:38:kworker.0 Request processed in 73us (301us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320005.854356:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1879 00000100:00100000:0.0:1767320005.854388:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314688 00000100:00100000:0.0:1767320005.854392:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314688 00000100:00100000:0.0:1767320005.854439:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314944 00000100:00100000:0.0:1767320005.854442:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314944 00000100:00100000:2.0:1767320005.854608:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314432 00000100:00100000:2.0:1767320005.854612:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314432 00000100:00100000:2.0:1767320005.854632:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985314816 00000100:00100000:2.0:1767320005.854635:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985314816 00000100:00100000:2.0:1767320005.854642:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315072 00000100:00100000:2.0:1767320005.854644:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315072 00000100:00100000:2.0:1767320005.854655:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4962 00000100:00100000:2.0:1767320005.854665:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985314688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320005.854720:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985314688:12345-0@lo:400:kworker.0 Request processed in 58us (980us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320005.854727:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4962 00000100:00100000:2.0:1767320005.854741:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5554 00000100:00100000:2.0:1767320005.854745:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985314432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320005.854756:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320005.854781:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985314432:12345-0@lo:38:kworker.0 Request processed in 34us (778us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320005.854786:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5554 00000100:00100000:2.0:1767320005.854799:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5555 00000100:00100000:2.0:1767320005.854803:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985314816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320005.854812:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320005.854832:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985314816:12345-0@lo:38:kworker.0 Request processed in 28us (735us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320005.854838:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5555 00000100:00100000:2.0:1767320005.854849:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5556 00000100:00100000:2.0:1767320005.854852:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985315072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320005.854860:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320005.854879:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985315072:12345-0@lo:38:kworker.0 Request processed in 25us (741us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320005.854884:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5556 00000100:00100000:2.0:1767320005.854923:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985314688:0@lo:400:kworker.0 00000100:00080000:2.0:1767320005.854978:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.854983:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320005.854987:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985314432:0@lo:38:kworker.0 00000100:00080000:2.0:1767320005.854994:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.854997:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320005.855000:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985314560:0@lo:38:kworker.0 00000100:00080000:2.0:1767320005.855007:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.855010:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320005.855013:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985314816:0@lo:38:kworker.0 00000100:00080000:2.0:1767320005.855020:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.855023:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320005.855025:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985315072:0@lo:38:kworker.0 00000100:00080000:2.0:1767320005.855060:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320005.855139:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4963 00000100:00100000:0.0:1767320005.855147:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce187feb850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985314944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320005.855212:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce187feb850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985314944:12345-0@lo:400:kworker.0 Request processed in 66us (1372us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320005.855230:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4963 00000100:00100000:3.0:1767320005.855743:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315200 00000100:00100000:3.0:1767320005.855751:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315200 00000100:00100000:3.0:1767320005.855790:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315328 00000100:00100000:3.0:1767320005.855793:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315328 00000100:00100000:3.0:1767320005.855817:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3237 00000100:00100000:3.0:1767320005.855824:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985315200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320005.855874:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985315200:12345-0@lo:400:kworker.0 Request processed in 50us (3629us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320005.855880:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3237 00000100:00100000:3.0:1767320005.855895:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3238 00000100:00100000:3.0:1767320005.855901:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985315328:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320005.855933:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985315328:12345-0@lo:400:kworker.0 Request processed in 32us (3575us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320005.855939:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3238 00000100:00100000:3.0:1767320005.855996:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985315328:0@lo:400:kworker.0 00000100:00100000:3.0:1767320005.856010:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985315200:0@lo:400:kworker.0 00000100:00100000:3.0:1767320005.856070:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985314944:0@lo:400:kworker.0 00000100:00080000:2.0:1767320005.857986:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320005.857999:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320006.363544:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320006.363575:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985315456:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320006.363664:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320006.363907:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320006.363919:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985315584:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320006.363951:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320006.364195:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315456 00000100:00100000:1.0:1767320006.364235:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315456 00000100:00100000:1.0:1767320006.364296:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315584 00000100:00100000:1.0:1767320006.364300:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315584 00000100:00100000:1.0:1767320006.364485:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2426 00000100:00100000:1.0:1767320006.364497:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985315456:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320006.364535:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320006.364595:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985315456:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (931us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320006.364604:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2426 00000100:00100000:1.0:1767320006.364623:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2427 00000100:00100000:1.0:1767320006.364630:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985315584:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320006.364648:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320006.364678:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985315584:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (727us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320006.364683:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2427 00000100:00100000:2.0:1767320006.369633:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985315456:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320006.369727:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985315584:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320006.496504:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320007.517827:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320008.540198:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320009.564495:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320010.589094:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320010.922360:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320010.922675:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920592896 00000100:00100000:3.0:1767320010.922688:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320010.922696:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920592896 00000100:00100000:2.0:1767320010.922849:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920593024 00000100:00100000:2.0:1767320010.922855:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920593024 00000100:00080000:3.0:1767320010.922951:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320010.925009:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4964 00000100:00100000:0.0:1767320010.925043:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920593024:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320010.925208:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920593024:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 183us (2527us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320010.925221:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4964 00000100:00100000:0.0:1767320010.925264:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5557 00000100:00100000:0.0:1767320010.925271:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920592896:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320010.925308:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320010.925343:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920592896:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 72us (2993us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320010.925350:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5557 00000100:00080000:1.0:1767320010.971313:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.971324:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320010.971327:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.971331:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.971336:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.971344:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.971348:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.971356:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320010.971395:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985315712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320010.971421:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.971425:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.971427:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.971430:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.971432:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.971436:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320010.971448:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7b80 x1853163985315840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320010.971492:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.971496:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.971498:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.971501:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.971503:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.971506:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320010.971517:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4e00 x1853163985316096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320010.971533:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.971536:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.971538:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.971541:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.971543:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.971546:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320010.971555:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7800 x1853163985316352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320010.971609:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985316608:0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.971674:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320010.971739:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320010.971746:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985316480:0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.971774:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320010.971825:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985316608 00000100:00100000:1.0:1767320010.971834:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985316608 00000100:00100000:2.0:1767320010.971840:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985315712:0@lo:38:kworker.0 00000100:00100000:1.0:1767320010.971895:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985316480 00000100:00100000:1.0:1767320010.971901:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985316480 00000100:00100000:2.0:1767320010.971930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320010.971937:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3239 00000100:00100000:1.0:1767320010.971949:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985316608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320010.971991:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985315840:0@lo:38:kworker.0 00000100:00100000:1.0:1767320010.972026:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985316608:12345-0@lo:400:kworker.0 Request processed in 76us (347us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320010.972036:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320010.972045:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3239 00000100:00100000:1.0:1767320010.972089:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3240 00000100:00100000:2.0:1767320010.972098:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985316096:0@lo:38:kworker.0 00000100:00100000:1.0:1767320010.972110:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985316480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320010.972144:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320010.972195:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985316352:0@lo:38:kworker.0 00000100:00100000:1.0:1767320010.972212:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985316480:12345-0@lo:400:kworker.0 Request processed in 111us (435us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320010.972221:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3240 00000100:00100000:2.0:1767320010.972232:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320010.972259:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320010.972268:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320010.972287:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315712 00000100:00100000:1.0:1767320010.972291:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315712 00000100:00100000:1.0:1767320010.972310:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985316096 00000100:00100000:1.0:1767320010.972313:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985316096 00000100:00100000:1.0:1767320010.972324:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985316352 00000100:00100000:1.0:1767320010.972326:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985316352 00000100:00100000:1.0:1767320010.972339:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5558 00000100:00100000:1.0:1767320010.972343:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985315712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320010.972362:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320010.972406:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985315712:12345-0@lo:38:kworker.0 Request processed in 61us (479us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320010.972412:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5558 00000100:00100000:1.0:1767320010.972423:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5559 00000100:00100000:1.0:1767320010.972427:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985316096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320010.972435:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320010.972466:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985316096:12345-0@lo:38:kworker.0 Request processed in 38us (324us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320010.972472:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5559 00000100:00100000:1.0:1767320010.972484:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5560 00000100:00100000:1.0:1767320010.972488:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985316352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320010.972496:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320010.972519:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985316352:12345-0@lo:38:kworker.0 Request processed in 29us (290us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320010.972524:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5560 00000100:00100000:1.0:1767320010.972573:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985316480:0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.972583:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985316608:0@lo:400:kworker.0 00000100:00100000:2.0:1767320010.972682:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315840 00000100:00100000:2.0:1767320010.972691:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315840 00000100:00100000:0.0:1767320010.972723:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985316224:0@lo:400:kworker.0 00000100:00100000:2.0:1767320010.972745:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1880 00000100:00100000:2.0:1767320010.972754:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985315840:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320010.972763:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767320010.972775:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320010.972796:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320010.972806:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985315968:0@lo:400:kworker.0 00000100:00100000:2.0:1767320010.972833:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985315840:12345-0@lo:38:kworker.0 Request processed in 76us (797us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320010.972846:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1880 00000100:00100000:0.0:1767320010.972852:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320010.972899:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985316224 00000100:00100000:0.0:1767320010.972905:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985316224 00000100:00080000:2.0:1767320010.972907:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320010.972917:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.972926:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985315712:0@lo:38:kworker.0 00000100:00080000:2.0:1767320010.972945:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320010.972956:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.972960:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985315840:0@lo:38:kworker.0 00000100:00100000:0.0:1767320010.972972:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985315968 00000100:00080000:2.0:1767320010.972979:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320010.972980:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985315968 00000100:00080000:2.0:1767320010.972989:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.972995:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985316096:0@lo:38:kworker.0 00000100:00080000:2.0:1767320010.973016:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320010.973020:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4965 00000100:00080000:2.0:1767320010.973025:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.973031:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985316352:0@lo:38:kworker.0 00000100:00100000:0.0:1767320010.973041:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985316224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320010.973092:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985316224:12345-0@lo:400:kworker.0 Request processed in 61us (331us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320010.973099:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4965 00000100:00080000:2.0:1767320010.973109:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320010.973136:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4966 00000100:00100000:0.0:1767320010.973152:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985315968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320010.973227:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985315968:12345-0@lo:400:kworker.0 Request processed in 80us (382us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320010.973234:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4966 00000100:00100000:0.0:1767320010.973267:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985315968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320010.973277:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985316224:0@lo:400:kworker.0 00000100:00100000:3.0:1767320011.483640:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985316736:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320011.483708:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320011.483813:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985316736 00000100:00100000:1.0:1767320011.483822:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985316736 00000100:00100000:1.0:1767320011.483870:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2428 00000100:00100000:1.0:1767320011.483880:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985316736:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320011.483920:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320011.483963:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985316736:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (254us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320011.483979:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2428 00000100:00100000:3.0:1767320011.484078:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985316736:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320011.484093:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320011.484099:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985316864:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320011.484147:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320011.484518:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985316864 00000100:00100000:1.0:1767320011.484524:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985316864 00000100:00100000:1.0:1767320011.484555:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2429 00000100:00100000:1.0:1767320011.484562:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985316864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320011.484577:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320011.484621:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985316864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 59us (472us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320011.484628:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2429 00000100:00100000:0.0:1767320011.484730:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985316864:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320011.611391:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320012.635406:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320013.663803:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320014.683361:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320015.067308:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319990/real 1767319990] req@ffff8ce1a61c5500 x1853163985310464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320015 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320015.067342:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61c5500 x1853163985310464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320015 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320015.067362:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320015.067387:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320015.067391:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985310464:0@lo:250:kworker.0 00000100:00080000:1.0:1767320015.067560:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320015.067565:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320015.067567:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320015.067572:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5238 00000100:00080000:1.0:1767320015.067576:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320015.067582:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320015.067614:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985316992/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320015.067648:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320015.067651:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320015.067653:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320015.067657:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5259 00000100:00080000:1.0:1767320015.067659:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320015.067663:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320015.067677:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6a00 x1853163985317120/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320015.067686:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320015.067689:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320015.067690:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320015.067693:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5259 00000100:00080000:1.0:1767320015.067696:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320015.067698:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320015.067710:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7100 x1853163985317248/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320015.067753:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320015.067757:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320015.067758:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320015.067761:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5259 00000100:00080000:1.0:1767320015.067764:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320015.067766:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320015.067776:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6d80 x1853163985317504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320015.067790:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320015.067793:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320015.067794:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320015.067797:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5259 00000100:00080000:1.0:1767320015.067799:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320015.067802:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320015.067810:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985317760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320015.067854:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985318016:0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.067910:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320015.067975:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320015.067981:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985317888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.068002:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320015.068028:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318016 00000100:00100000:1.0:1767320015.068034:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318016 00000100:00100000:1.0:1767320015.068104:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985317888 00000100:00100000:1.0:1767320015.068107:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985317888 00000100:00100000:1.0:1767320015.068125:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3241 00000100:00100000:1.0:1767320015.068132:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985318016:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.068201:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985318016:12345-0@lo:400:kworker.0 Request processed in 68us (289us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320015.068208:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3241 00000100:00100000:1.0:1767320015.068232:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3242 00000100:00100000:1.0:1767320015.068239:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985317888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.068265:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985317888:12345-0@lo:400:kworker.0 Request processed in 28us (262us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320015.068271:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3242 00000100:00100000:1.0:1767320015.068311:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985317888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.068320:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985318016:0@lo:400:kworker.0 00000100:00080000:2.0:1767320015.068386:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320015.068389:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320015.068398:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985316992:0@lo:250:kworker.0 00000400:00000100:2.0:1767320015.068420:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985316992 offset 0 length 520: 4 00000100:00100000:2.0:1767320015.068438:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985317120:0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.068460:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320015.068482:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985317248:0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.068501:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320015.068517:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985317504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.068532:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320015.068543:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985317760:0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.068559:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320015.068584:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985317248 00000100:00100000:2.0:1767320015.068587:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985317248 00000100:00100000:2.0:1767320015.068608:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1881 00000100:00100000:2.0:1767320015.068612:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985317248:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320015.068631:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320015.069129:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985317120 00000100:00100000:3.0:1767320015.069134:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985317120 00000100:00100000:3.0:1767320015.069157:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985317504 00000100:00100000:3.0:1767320015.069159:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985317504 00000100:00100000:3.0:1767320015.069210:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985317760 00000100:00100000:3.0:1767320015.069213:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985317760 00000100:00100000:3.0:1767320015.069229:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5561 00000100:00100000:3.0:1767320015.069235:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985317120:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320015.069246:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320015.069279:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985317120:12345-0@lo:38:kworker.0 Request processed in 44us (818us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320015.069285:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5561 00000100:00100000:3.0:1767320015.069299:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5562 00000100:00100000:3.0:1767320015.069303:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985317504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320015.069312:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320015.069322:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985317248:12345-0@lo:38:kworker.0 Request processed in 47us (158us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320015.069329:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1881 00000100:00100000:3.0:1767320015.069352:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985317504:12345-0@lo:38:kworker.0 Request processed in 47us (818us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320015.069355:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5563 00000100:00100000:3.0:1767320015.069358:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5562 00000100:00100000:2.0:1767320015.069359:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985317760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320015.069369:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320015.069382:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985317376:0@lo:400:kworker.0 00000100:00100000:2.0:1767320015.069393:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985317760:12345-0@lo:38:kworker.0 Request processed in 33us (834us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320015.069398:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5563 00000100:00100000:3.0:1767320015.069403:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320015.069409:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985317376 00000100:00100000:2.0:1767320015.069413:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985317376 00000100:00100000:3.0:1767320015.069424:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320015.069430:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985317632:0@lo:400:kworker.0 00000100:00100000:2.0:1767320015.069437:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4967 00000100:00100000:2.0:1767320015.069443:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985317376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320015.069447:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320015.069465:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985317632 00000100:00100000:3.0:1767320015.069469:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985317632 00000100:00100000:2.0:1767320015.069473:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985317376:12345-0@lo:400:kworker.0 Request processed in 31us (68us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320015.069479:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4967 00000100:00100000:3.0:1767320015.069488:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4968 00000100:00100000:3.0:1767320015.069493:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985317632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320015.069519:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985317632:12345-0@lo:400:kworker.0 Request processed in 26us (70us total) trans 0 rc 0/0 00000100:00080000:0.0:1767320015.069570:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320015.069593:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320015.069598:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320015.069601:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985317120:0@lo:38:kworker.0 00000100:00080000:0.0:1767320015.069609:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320015.069612:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320015.069615:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985317248:0@lo:38:kworker.0 00000100:00080000:0.0:1767320015.069622:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320015.069625:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320015.069627:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985317504:0@lo:38:kworker.0 00000100:00080000:0.0:1767320015.069634:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320015.069638:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320015.069641:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985317760:0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.070315:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985317632:0@lo:400:kworker.0 00000100:00100000:2.0:1767320015.070331:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985317376:0@lo:400:kworker.0 00000100:00100000:3.0:1767320015.071427:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4968 00010000:00010000:2.0:1767320015.711446:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320016.049464:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320016.049492:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767320016.049516:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920593408 offset 0 length 520: 4 00000100:00100000:2.0:1767320016.049644:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920593280 00000100:00100000:2.0:1767320016.049656:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920593280 00000100:00100000:2.0:1767320016.049731:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920593152 00000100:00100000:2.0:1767320016.049734:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920593152 00000100:00100000:2.0:1767320016.049756:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4969 00000100:00100000:2.0:1767320016.049766:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920593280:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320016.049843:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920593280:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (386us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320016.049851:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4969 00000100:00100000:2.0:1767320016.049869:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5564 00000100:00100000:2.0:1767320016.049873:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920593152:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320016.049891:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320016.049913:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920593152:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 39us (419us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320016.049919:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5564 00000100:00080000:0.0:1767320016.050205:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320016.603427:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985318144:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320016.603498:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320016.603550:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320016.603557:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985318272:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320016.603579:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320016.603623:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318144 00000100:00100000:3.0:1767320016.603631:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318144 00000100:00100000:3.0:1767320016.603683:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318272 00000100:00100000:3.0:1767320016.603687:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318272 00000100:00100000:3.0:1767320016.603717:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2430 00000100:00100000:3.0:1767320016.603727:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985318144:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320016.603735:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2431 00000100:00100000:1.0:1767320016.603744:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985318272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320016.603756:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:1.0:1767320016.603768:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320016.603799:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985318144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320016.603806:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2430 00000100:00100000:1.0:1767320016.609099:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985318272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 5345us (5507us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320016.609117:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2431 00000100:00100000:2.0:1767320016.615837:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985318272:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320016.615878:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985318144:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320016.731888:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320017.755327:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320018.779507:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320019.804020:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320020.187367:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320020.187377:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320020.187381:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320020.187384:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320020.187388:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320020.187396:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5263 00000100:00080000:1.0:1767320020.187401:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320020.187406:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320020.187438:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985318400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320020.187463:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320020.187466:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320020.187468:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320020.187471:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5263 00000100:00080000:1.0:1767320020.187473:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320020.187476:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320020.187486:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7480 x1853163985318528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320020.187534:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320020.187538:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320020.187540:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320020.187543:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5263 00000100:00080000:1.0:1767320020.187546:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320020.187549:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320020.187560:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4380 x1853163985318784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320020.187577:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320020.187580:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320020.187582:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320020.187585:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5263 00000100:00080000:1.0:1767320020.187587:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320020.187590:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320020.187598:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985319040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320020.187650:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985319296:0@lo:400:kworker.0 00000100:00100000:1.0:1767320020.187705:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320020.187759:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320020.187765:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985319168:0@lo:400:kworker.0 00000100:00100000:1.0:1767320020.187785:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320020.187811:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319296 00000100:00100000:1.0:1767320020.187817:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319296 00000100:00100000:1.0:1767320020.187858:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319168 00000100:00100000:1.0:1767320020.187861:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319168 00000100:00100000:1.0:1767320020.187879:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3243 00000100:00100000:1.0:1767320020.187887:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985319296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320020.187944:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985319296:12345-0@lo:400:kworker.0 Request processed in 58us (238us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320020.187963:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3243 00000100:00100000:1.0:1767320020.187979:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3244 00000100:00100000:1.0:1767320020.187986:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985319168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320020.188012:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985319168:12345-0@lo:400:kworker.0 Request processed in 27us (226us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320020.188019:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3244 00000100:00080000:1.0:1767320020.188034:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320020.188038:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320020.188071:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985319168:0@lo:400:kworker.0 00000100:00100000:1.0:1767320020.188081:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985319296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.188149:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985318400:0@lo:38:kworker.0 00000100:00100000:2.0:1767320020.188193:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985318656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.188194:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.188217:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985318528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320020.188219:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.188237:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320020.188237:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320020.188242:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985318912:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.188257:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985318784:0@lo:38:kworker.0 00000100:00100000:2.0:1767320020.188258:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.188274:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.188288:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985319040:0@lo:38:kworker.0 00000100:00100000:2.0:1767320020.188294:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318528 00000100:00100000:2.0:1767320020.188297:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318528 00000100:00100000:0.0:1767320020.188305:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320020.188318:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1882 00000100:00100000:2.0:1767320020.188323:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985318528:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320020.188329:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318400 00000100:00100000:0.0:1767320020.188332:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318400 00010000:00020000:2.0:1767320020.188339:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320020.188347:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318656 00000100:00100000:0.0:1767320020.188349:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318656 00000100:00100000:0.0:1767320020.188368:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318912 00000100:00100000:2.0:1767320020.188369:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985318528:12345-0@lo:38:kworker.0 Request processed in 45us (131us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320020.188371:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318912 00000100:00100000:2.0:1767320020.188378:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1882 00000100:00100000:0.0:1767320020.188382:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985318784 00000100:00100000:0.0:1767320020.188384:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985318784 00000100:00100000:0.0:1767320020.188391:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319040 00000100:00100000:0.0:1767320020.188393:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319040 00000100:00100000:0.0:1767320020.188401:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4970 00000100:00080000:2.0:1767320020.188401:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320020.188405:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320020.188406:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985318656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320020.188408:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985318528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320020.188428:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4971 00000100:00100000:2.0:1767320020.188435:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985318912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.188438:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985318656:12345-0@lo:400:kworker.0 Request processed in 32us (218us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320020.188443:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4970 00000100:00100000:0.0:1767320020.188456:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5565 00000100:00100000:2.0:1767320020.188458:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985318912:12345-0@lo:400:kworker.0 Request processed in 25us (199us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320020.188460:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985318400:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320020.188464:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4971 00010000:00020000:0.0:1767320020.188469:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320020.188475:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5566 00000100:00100000:2.0:1767320020.188479:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985318784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320020.188487:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320020.188492:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985318400:12345-0@lo:38:kworker.0 Request processed in 31us (297us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320020.188497:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5565 00000100:00100000:2.0:1767320020.188503:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985318784:12345-0@lo:38:kworker.0 Request processed in 24us (229us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320020.188507:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5567 00000100:00100000:2.0:1767320020.188510:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5566 00000100:00100000:0.0:1767320020.188511:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985319040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320020.188519:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320020.188529:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985318912:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.188537:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985319040:12345-0@lo:38:kworker.0 Request processed in 26us (231us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320020.188537:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985318656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.188542:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5567 00000100:00080000:2.0:1767320020.188556:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320020.188559:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00080000:0.0:1767320020.188561:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320020.188562:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985318400:0@lo:38:kworker.0 00000100:00080000:2.0:1767320020.188569:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320020.188572:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320020.188574:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985318784:0@lo:38:kworker.0 00000100:00080000:2.0:1767320020.188581:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320020.188584:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320020.188587:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985319040:0@lo:38:kworker.0 00010000:00010000:1.0:1767320020.827402:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320021.161758:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320021.161888:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920593664 00000100:00100000:1.0:1767320021.161902:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920593664 00000100:00100000:1.0:1767320021.162026:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4972 00000100:00100000:1.0:1767320021.162039:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920593664:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320021.162143:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920593664:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 106us (400us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320021.162153:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4972 00000100:00080000:2.0:1767320021.162468:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320021.162639:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320021.162701:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920593536 00000100:00100000:3.0:1767320021.162707:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920593536 00000100:00100000:3.0:1767320021.162741:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5568 00000100:00100000:3.0:1767320021.162748:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920593536:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320021.162774:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320021.162824:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920593536:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 76us (186us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320021.162831:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5568 00000100:00100000:3.0:1767320021.723594:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985319552:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320021.723708:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320021.723790:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985319424:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320021.723824:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320021.724875:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319552 00000100:00100000:2.0:1767320021.724891:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319552 00000100:00100000:2.0:1767320021.724972:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319424 00000100:00100000:2.0:1767320021.724977:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319424 00000100:00100000:2.0:1767320021.725017:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2432 00000100:00100000:2.0:1767320021.725028:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985319552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320021.725070:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320021.725145:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985319552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (1439us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320021.725154:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2432 00000100:00100000:3.0:1767320021.725176:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2433 00000100:00100000:3.0:1767320021.725186:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985319424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320021.725205:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320021.725224:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985319552:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320021.725238:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985319424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (1414us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320021.725244:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2433 00000100:00100000:2.0:1767320021.725273:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985319424:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320021.851523:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320022.876939:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320023.899412:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320024.925801:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320025.307734:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320025.307750:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320025.307754:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320025.307759:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320025.307765:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320025.307775:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5268 00000100:00080000:2.0:1767320025.307781:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320025.307789:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320025.307865:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd8700 x1853163985319680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320025.307888:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320025.307892:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320025.307894:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320025.307898:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5268 00000100:00080000:2.0:1767320025.307900:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320025.307904:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320025.307918:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb800 x1853163985319808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320025.307965:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320025.307969:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320025.307971:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320025.307975:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5268 00000100:00080000:2.0:1767320025.307977:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320025.307981:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320025.307996:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939c00 x1853163985320064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320025.308012:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320025.308015:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320025.308017:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320025.308021:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5268 00000100:00080000:2.0:1767320025.308023:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320025.308027:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320025.308036:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a680 x1853163985320320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320025.308096:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985320576:0@lo:400:kworker.0 00000100:00100000:2.0:1767320025.308204:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320025.308276:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320576 00000100:00100000:2.0:1767320025.308278:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320025.308284:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985320448:0@lo:400:kworker.0 00000100:00100000:1.0:1767320025.308287:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320576 00000100:00100000:2.0:1767320025.308318:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320025.308352:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320448 00000100:00100000:1.0:1767320025.308356:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320448 00000100:00080000:2.0:1767320025.308370:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320025.308375:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320025.308381:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3245 00000100:00100000:2.0:1767320025.308389:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985319936:0@lo:400:kworker.0 00000100:00100000:1.0:1767320025.308391:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985320576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320025.308417:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320025.308444:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320025.308450:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985320192:0@lo:400:kworker.0 00000100:00100000:1.0:1767320025.308453:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985320576:12345-0@lo:400:kworker.0 Request processed in 65us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320025.308461:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3245 00000100:00100000:2.0:1767320025.308469:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320025.308480:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3246 00000100:00100000:1.0:1767320025.308486:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985320448:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320025.308511:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985320448:12345-0@lo:400:kworker.0 Request processed in 26us (192us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320025.308517:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3246 00000100:00100000:2.0:1767320025.308524:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319936 00000100:00100000:2.0:1767320025.308528:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319936 00000100:00100000:1.0:1767320025.308555:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985320448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320025.308560:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320192 00000100:00100000:2.0:1767320025.308563:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320192 00000100:00100000:1.0:1767320025.308567:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985320576:0@lo:400:kworker.0 00000100:00100000:2.0:1767320025.308583:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4973 00000100:00100000:2.0:1767320025.308588:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985319936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320025.308631:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985319936:12345-0@lo:400:kworker.0 Request processed in 43us (214us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320025.308637:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4973 00000100:00100000:2.0:1767320025.308651:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4974 00000100:00100000:2.0:1767320025.308657:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985320192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320025.308683:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985320192:12345-0@lo:400:kworker.0 Request processed in 26us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320025.308691:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985320192:0@lo:400:kworker.0 00000100:00100000:1.0:1767320025.308699:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985319936:0@lo:400:kworker.0 00000100:00100000:0.0:1767320025.308930:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985319680:0@lo:38:kworker.0 00000100:00100000:0.0:1767320025.308961:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320025.308977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985319808:0@lo:38:kworker.0 00000100:00100000:0.0:1767320025.309000:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320025.309020:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985320064:0@lo:38:kworker.0 00000100:00100000:0.0:1767320025.309036:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320025.309047:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985320320:0@lo:38:kworker.0 00000100:00100000:0.0:1767320025.309062:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320025.309089:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319808 00000100:00100000:0.0:1767320025.309092:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319808 00000100:00100000:0.0:1767320025.309117:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1883 00000100:00100000:0.0:1767320025.309122:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985319808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320025.309144:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320025.309192:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985319808:12345-0@lo:38:kworker.0 Request processed in 68us (190us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320025.309199:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1883 00000100:00100000:0.0:1767320025.309219:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985319680 00000100:00100000:0.0:1767320025.309223:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985319680 00000100:00100000:1.0:1767320025.309443:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320064 00000100:00100000:1.0:1767320025.309448:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320064 00000100:00100000:3.0:1767320025.313549:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320320 00000100:00100000:3.0:1767320025.313574:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320320 00000100:00100000:3.0:1767320025.313677:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5569 00000100:00100000:3.0:1767320025.313687:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985320320:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320025.313718:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320025.313777:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4974 00000100:00100000:3.0:1767320025.313815:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985320320:12345-0@lo:38:kworker.0 Request processed in 129us (4750us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320025.313822:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5569 00000100:00100000:0.0:1767320025.314065:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5570 00000100:00100000:0.0:1767320025.314071:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985319680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320025.314084:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320025.314119:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985319680:12345-0@lo:38:kworker.0 Request processed in 47us (5158us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320025.314125:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5570 00000100:00080000:0.0:1767320025.314208:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320025.314215:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320025.314220:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985319680:0@lo:38:kworker.0 00000100:00080000:0.0:1767320025.314235:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320025.314239:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320025.314242:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985319808:0@lo:38:kworker.0 00000100:00080000:0.0:1767320025.314252:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320025.314256:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320025.314259:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985320320:0@lo:38:kworker.0 00000100:00080000:2.0:1767320025.314431:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320025.316060:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5571 00000100:00100000:1.0:1767320025.316235:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985320064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320025.316266:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320025.316370:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985320064:12345-0@lo:38:kworker.0 Request processed in 296us (7331us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320025.316377:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5571 00000100:00080000:0.0:1767320025.316400:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320025.316406:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320025.316409:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985320064:0@lo:38:kworker.0 00010000:00010000:0.0:1767320025.947421:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320026.281521:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320026.281621:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920593920 00000100:00100000:3.0:1767320026.281633:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920593920 00000100:00100000:3.0:1767320026.281719:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4975 00000100:00100000:3.0:1767320026.281730:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920593920:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320026.281819:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920593920:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (309us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320026.281827:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4975 00000100:00080000:2.0:1767320026.281940:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320026.286759:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320026.286875:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920593792 00000100:00100000:1.0:1767320026.286885:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920593792 00000100:00100000:1.0:1767320026.286931:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5572 00000100:00100000:1.0:1767320026.286939:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920593792:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320026.286963:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320026.287043:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920593792:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 102us (293us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320026.287051:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5572 00000100:00100000:2.0:1767320026.843565:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985320704:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320026.843645:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320026.843706:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320026.843712:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985320832:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320026.843733:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320026.843810:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320704 00000100:00100000:0.0:1767320026.843818:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320704 00000100:00100000:0.0:1767320026.843878:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320832 00000100:00100000:0.0:1767320026.843882:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320832 00000100:00100000:0.0:1767320026.843902:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2434 00000100:00100000:0.0:1767320026.843911:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985320704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320026.843941:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320026.843984:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985320704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (338us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320026.843991:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2434 00000100:00100000:0.0:1767320026.844009:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2435 00000100:00100000:0.0:1767320026.844016:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985320832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320026.844030:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320026.844054:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985320832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 39us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320026.844060:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2435 00000100:00100000:2.0:1767320026.844202:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985320832:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320026.844214:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985320704:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320026.971498:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320027.995431:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320029.020403:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320030.043976:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320030.428966:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320030.428975:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320030.428978:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320030.428982:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320030.428986:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320030.428993:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5273 00000100:00080000:1.0:1767320030.428997:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320030.429003:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320030.429039:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7800 x1853163985320960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320030.429066:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320030.429069:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320030.429071:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320030.429073:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5273 00000100:00080000:1.0:1767320030.429076:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320030.429079:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320030.429090:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5c00 x1853163985321088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320030.429128:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320030.429131:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320030.429132:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320030.429135:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5273 00000100:00080000:1.0:1767320030.429137:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320030.429140:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320030.429149:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6680 x1853163985321344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320030.429189:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320030.429193:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320030.429195:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320030.429198:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5273 00000100:00080000:1.0:1767320030.429200:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320030.429203:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320030.429213:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4700 x1853163985321600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320030.429265:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985321856:0@lo:400:kworker.0 00000100:00100000:0.0:1767320030.429311:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985320960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320030.429339:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320030.429339:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320030.429388:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985321216:0@lo:400:kworker.0 00000100:00100000:3.0:1767320030.429449:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985321472:0@lo:400:kworker.0 00000100:00100000:1.0:1767320030.431971:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320030.431988:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985321728:0@lo:400:kworker.0 00000100:00100000:2.0:1767320030.432973:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320030.433071:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321856 00000100:00100000:2.0:1767320030.433079:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321856 00000100:00100000:2.0:1767320030.433130:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3247 00000100:00100000:2.0:1767320030.433139:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985321856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320030.433204:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985321856:12345-0@lo:400:kworker.0 Request processed in 67us (3864us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320030.433211:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3247 00000100:00080000:2.0:1767320030.433233:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320030.433250:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985320960 00000100:00100000:2.0:1767320030.433253:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985320960 00000100:00100000:2.0:1767320030.433270:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321216 00000100:00100000:2.0:1767320030.433272:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321216 00000100:00100000:2.0:1767320030.433296:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4976 00000100:00100000:2.0:1767320030.433302:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985321216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320030.433334:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985321216:12345-0@lo:400:kworker.0 Request processed in 32us (365us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320030.433341:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4976 00000100:00100000:2.0:1767320030.433352:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5573 00000100:00100000:2.0:1767320030.433356:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985320960:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320030.433375:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320030.433402:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985320960:12345-0@lo:38:kworker.0 Request processed in 45us (4062us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320030.433408:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5573 00000100:00080000:2.0:1767320030.433420:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320030.433448:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985321216:0@lo:400:kworker.0 00000100:00080000:0.0:1767320030.437498:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320030.437499:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320030.437509:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320030.437514:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985320960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320030.437527:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985321088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320030.437568:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320030.437609:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985321344:0@lo:38:kworker.0 00000100:00100000:3.0:1767320030.437614:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321088 00000100:00100000:3.0:1767320030.437621:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321088 00000100:00100000:0.0:1767320030.437634:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320030.437660:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1884 00000100:00100000:3.0:1767320030.437667:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985321088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320030.437688:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320030.437852:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321472 00000100:00100000:2.0:1767320030.437858:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321472 00000100:00100000:2.0:1767320030.437902:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321344 00000100:00100000:2.0:1767320030.437905:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321344 00000100:00100000:2.0:1767320030.437920:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4977 00000100:00100000:2.0:1767320030.437927:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985321472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320030.438035:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985321600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320030.438755:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320030.438816:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985321856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320030.438817:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985321472:12345-0@lo:400:kworker.0 Request processed in 888us (1324us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320030.438829:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4977 00000100:00100000:2.0:1767320030.438852:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5574 00000100:00100000:2.0:1767320030.438857:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985321344:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320030.438866:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321728 00010000:00020000:2.0:1767320030.438871:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320030.438872:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321728 00000100:00100000:2.0:1767320030.438904:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985321344:12345-0@lo:38:kworker.0 Request processed in 45us (1269us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320030.438913:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5574 00000100:00080000:2.0:1767320030.438937:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320030.438942:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320030.438994:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985321472:0@lo:400:kworker.0 00000100:00100000:1.0:1767320030.439040:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3248 00000100:00100000:1.0:1767320030.439051:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985321728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320030.439093:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985321728:12345-0@lo:400:kworker.0 Request processed in 45us (340us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320030.439313:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320030.439394:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320030.439399:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320030.439403:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985321344:0@lo:38:kworker.0 00000100:00100000:0.0:1767320030.439435:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985321728:0@lo:400:kworker.0 00000100:00100000:2.0:1767320030.439483:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321600 00000100:00100000:2.0:1767320030.439488:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321600 00000100:00100000:2.0:1767320030.439552:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5575 00000100:00100000:2.0:1767320030.439557:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985321600:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320030.439568:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320030.439598:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985321600:12345-0@lo:38:kworker.0 Request processed in 40us (285us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320030.439603:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5575 00000100:00080000:0.0:1767320030.439786:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320030.439790:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320030.439793:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985321600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320030.449745:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3248 00000100:00100000:3.0:1767320030.450189:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985321088:12345-0@lo:38:kworker.0 Request processed in 12518us (12617us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320030.450201:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1884 00000100:00080000:0.0:1767320030.450724:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320030.450732:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320030.450736:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985321088:0@lo:38:kworker.0 00010000:00010000:0.0:1767320031.069561:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320031.403665:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320031.403739:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920594176 00000100:00100000:2.0:1767320031.403778:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920594176 00000100:00080000:0.0:1767320031.403833:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320031.403868:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4978 00000100:00100000:2.0:1767320031.403881:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920594176:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320031.404435:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920594176:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 555us (783us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320031.404444:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4978 00000100:00100000:2.0:1767320031.405760:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320031.405817:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920594048 00000100:00100000:0.0:1767320031.405826:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920594048 00000100:00100000:0.0:1767320031.405944:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5576 00000100:00100000:0.0:1767320031.405953:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920594048:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320031.405975:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320031.406046:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920594048:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (302us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320031.406055:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5576 00000100:00100000:2.0:1767320031.964381:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985321984:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320031.964450:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320031.964486:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320031.964493:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985322112:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320031.964515:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320031.964542:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985321984 00000100:00100000:2.0:1767320031.964550:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985321984 00000100:00100000:2.0:1767320031.964593:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985322112 00000100:00100000:2.0:1767320031.964597:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985322112 00000100:00100000:2.0:1767320031.964616:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2436 00000100:00100000:2.0:1767320031.964624:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985321984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320031.964652:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320031.964689:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985321984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (239us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320031.964697:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2436 00000100:00100000:2.0:1767320031.964714:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2437 00000100:00100000:2.0:1767320031.964720:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985322112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320031.964746:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320031.964771:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985322112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (255us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320031.964778:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2437 00000100:00100000:2.0:1767320031.964823:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985322112:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320031.964834:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985321984:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320032.091437:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320033.115503:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320034.140374:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320035.163863:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320035.547593:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320035.547603:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320035.547608:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320035.547612:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320035.547615:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320035.547623:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5278 00000100:00080000:2.0:1767320035.547627:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320035.547634:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320035.547686:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a680 x1853163985322240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320035.547715:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320035.547719:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320035.547721:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320035.547724:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5278 00000100:00080000:2.0:1767320035.547726:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320035.547730:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320035.547742:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938a80 x1853163985322368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320035.547821:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767320035.547841:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320035.547845:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320035.547847:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320035.547852:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5278 00000100:00080000:2.0:1767320035.547855:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320035.547859:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320035.547873:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593ad80 x1853163985322624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320035.547879:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985322240:0@lo:38:kworker.0 00000100:00100000:3.0:1767320035.547956:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320035.547968:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985322752:0@lo:400:kworker.0 00000100:00100000:3.0:1767320035.548019:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320035.548112:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985322752 00000100:00100000:3.0:1767320035.548119:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985322752 00000100:00100000:3.0:1767320035.548197:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4979 00000100:00100000:3.0:1767320035.548207:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985322752:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320035.548255:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985322752:12345-0@lo:400:kworker.0 Request processed in 51us (236us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320035.548262:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4979 00000100:00080000:3.0:1767320035.548284:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320035.548294:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985322496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320035.548316:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985322752:0@lo:400:kworker.0 00000100:00100000:1.0:1767320035.548329:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320035.548365:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985322496 00000100:00100000:1.0:1767320035.548369:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985322496 00000100:00100000:1.0:1767320035.548403:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4980 00000100:00100000:1.0:1767320035.548410:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985322496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320035.548456:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320035.548467:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985322496:12345-0@lo:400:kworker.0 Request processed in 44us (124us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320035.548474:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4980 00000100:00100000:0.0:1767320035.548481:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985322368:0@lo:38:kworker.0 00000100:00100000:1.0:1767320035.548494:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985322240 00000100:00100000:1.0:1767320035.548499:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985322240 00000100:00100000:0.0:1767320035.548510:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320035.548521:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5577 00000100:00100000:1.0:1767320035.548527:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985322240:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320035.548542:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985322624:0@lo:38:kworker.0 00000100:00100000:0.0:1767320035.548562:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320035.548587:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985322624 00000100:00100000:0.0:1767320035.548591:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985322624 00000100:00100000:0.0:1767320035.548606:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5578 00000100:00100000:0.0:1767320035.548623:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985322624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320035.548640:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320035.548680:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985322624:12345-0@lo:38:kworker.0 Request processed in 68us (116us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320035.548687:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5578 00000100:00080000:0.0:1767320035.548711:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320035.548716:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320035.548719:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985322624:0@lo:38:kworker.0 00000100:00100000:0.0:1767320035.548756:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985322496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320035.548799:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985322368 00000100:00100000:0.0:1767320035.548802:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985322368 00000100:00100000:0.0:1767320035.548822:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1885 00000100:00100000:0.0:1767320035.548826:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985322368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320035.548836:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320035.548862:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320035.548866:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985322368:12345-0@lo:38:kworker.0 Request processed in 36us (354us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320035.548873:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1885 00000100:00100000:1.0:1767320035.548900:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985322240:12345-0@lo:38:kworker.0 Request processed in 371us (446us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767320035.548905:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320035.548910:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5577 00000100:00080000:0.0:1767320035.548915:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320035.548919:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985322240:0@lo:38:kworker.0 00000100:00080000:0.0:1767320035.548929:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320035.548932:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320035.548935:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985322368:0@lo:38:kworker.0 00000100:00080000:0.0:1767320035.548969:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320035.551454:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320035.551465:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320035.551468:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320035.551477:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5278 00000100:00080000:2.0:1767320035.551482:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320035.551489:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320035.551531:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7100 x1853163985322880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320035.551735:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985322880:0@lo:38:kworker.0 00000100:00100000:0.0:1767320035.551809:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320035.551899:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985322880 00000100:00100000:1.0:1767320035.551908:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985322880 00000100:00100000:1.0:1767320035.551964:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5579 00000100:00100000:1.0:1767320035.551972:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985322880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320035.551992:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320035.552212:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985323008:0@lo:400:kworker.0 00000100:00100000:2.0:1767320035.552243:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320035.552268:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320035.552274:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985323136:0@lo:400:kworker.0 00000100:00100000:2.0:1767320035.552295:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320035.552319:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985323008 00000100:00100000:2.0:1767320035.552322:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985323008 00000100:00100000:2.0:1767320035.552360:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985323136 00000100:00100000:2.0:1767320035.552363:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985323136 00000100:00100000:2.0:1767320035.552386:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3249 00000100:00100000:2.0:1767320035.552391:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985323008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320035.552433:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985323008:12345-0@lo:400:kworker.0 Request processed in 41us (188us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320035.552440:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3249 00000100:00100000:2.0:1767320035.552457:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3250 00000100:00100000:2.0:1767320035.552463:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985323136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320035.552490:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985323136:12345-0@lo:400:kworker.0 Request processed in 27us (195us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320035.552496:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3250 00000100:00080000:2.0:1767320035.552517:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320035.552520:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320035.552547:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985323136:0@lo:400:kworker.0 00000100:00100000:2.0:1767320035.552556:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985323008:0@lo:400:kworker.0 00000100:00100000:1.0:1767320035.555727:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985322880:12345-0@lo:38:kworker.0 Request processed in 3742us (3905us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320035.555743:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5579 00000100:00080000:0.0:1767320035.563435:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320035.563449:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320035.563454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985322880:0@lo:38:kworker.0 00010000:00010000:2.0:1767320036.187674:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320036.524001:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320036.524138:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920594432 00000100:00100000:1.0:1767320036.524146:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920594432 00000100:00100000:1.0:1767320036.524272:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4981 00000100:00100000:1.0:1767320036.524282:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920594432:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320036.524354:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920594432:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (364us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320036.524360:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4981 00000100:00080000:2.0:1767320036.524498:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320036.525264:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320036.525312:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920594304 00000100:00100000:1.0:1767320036.525317:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920594304 00000100:00100000:1.0:1767320036.525363:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5580 00000100:00100000:1.0:1767320036.525368:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920594304:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320036.525387:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320036.525455:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920594304:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 87us (193us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320036.525461:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5580 00000100:00100000:3.0:1767320037.085509:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985323264:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320037.085581:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320037.085674:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985323264 00000100:00100000:2.0:1767320037.085683:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985323264 00000100:00080000:0.0:1767320037.085802:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320037.085813:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985323392:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320037.085839:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320037.086702:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985323392 00000100:00100000:2.0:1767320037.086710:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985323392 00000100:00100000:3.0:1767320037.086734:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2438 00000100:00100000:3.0:1767320037.086749:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985323264:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320037.086764:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2439 00000100:00100000:2.0:1767320037.086774:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985323392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320037.086781:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:2.0:1767320037.086795:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767320037.086853:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320037.087378:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985323392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 604us (1535us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320037.087387:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2439 00000100:00100000:1.0:1767320037.087445:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985323392:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320037.087627:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985323264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 882us (2044us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320037.087635:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2438 00000100:00100000:0.0:1767320037.087708:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985323264:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320037.212884:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320038.235505:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320039.261626:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320040.283440:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767320040.619404:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920594560 offset 0 length 520: 4 00000100:00100000:2.0:1767320040.624721:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320040.624831:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920594688 00000100:00100000:1.0:1767320040.624841:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920594688 00000100:00100000:1.0:1767320040.624880:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5581 00000100:00100000:1.0:1767320040.624888:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920594688:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320040.624910:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320040.624976:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920594688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 87us (264us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320040.624983:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5581 00000100:00100000:0.0:1767320040.635007:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320040.635084:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920594816 00000100:00100000:3.0:1767320040.635094:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920594816 00000100:00100000:3.0:1767320040.635198:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4982 00000100:00100000:3.0:1767320040.635208:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920594816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320040.635282:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920594816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (284us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320040.635290:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4982 00000100:00080000:1.0:1767320040.635506:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320040.667288:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320015/real 1767320015] req@ffff8ce1a54b5180 x1853163985316992/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320040 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320040.667321:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a54b5180 x1853163985316992/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320040 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320040.667339:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.667348:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320040.667352:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985316992:0@lo:250:kworker.0 00000100:00080000:3.0:1767320040.667489:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.667495:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320040.667497:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.667502:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5263 00000100:00080000:3.0:1767320040.667506:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.667517:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320040.667552:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985323520/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.667600:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.667606:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.667608:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.667611:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5283 00000100:00080000:3.0:1767320040.667614:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.667617:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320040.667630:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631f80 x1853163985323648/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.667638:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.667641:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.667642:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.667645:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5283 00000100:00080000:3.0:1767320040.667647:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.667650:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320040.667659:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631c00 x1853163985323776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.667712:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.667716:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.667717:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.667721:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5283 00000100:00080000:3.0:1767320040.667723:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.667726:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320040.667739:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632680 x1853163985324032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.667754:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.667757:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.667758:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.667761:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5283 00000100:00080000:3.0:1767320040.667763:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.667765:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320040.667775:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630000 x1853163985324288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320040.667830:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985324416:0@lo:400:kworker.0 00000100:00100000:3.0:1767320040.667908:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320040.667973:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320040.667979:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985324544:0@lo:400:kworker.0 00000100:00100000:3.0:1767320040.668002:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320040.668071:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985323904:0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.668099:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320040.668127:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320040.668132:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985324160:0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.668150:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320040.668240:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985323904 00000100:00100000:0.0:1767320040.668247:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985323904 00000100:00100000:0.0:1767320040.668294:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324160 00000100:00100000:0.0:1767320040.668297:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324160 00000100:00100000:0.0:1767320040.668314:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4983 00000100:00100000:0.0:1767320040.668321:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985323904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.668367:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985323904:12345-0@lo:400:kworker.0 Request processed in 46us (266us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320040.668375:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4983 00000100:00100000:0.0:1767320040.668392:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4984 00000100:00100000:0.0:1767320040.668399:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985324160:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.668425:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985324160:12345-0@lo:400:kworker.0 Request processed in 27us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320040.668431:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4984 00000100:00100000:0.0:1767320040.668469:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985324160:0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.668478:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985323904:0@lo:400:kworker.0 00000100:00080000:1.0:1767320040.668630:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320040.669251:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324416 00000100:00100000:2.0:1767320040.669257:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324416 00000100:00100000:2.0:1767320040.669320:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324544 00000100:00100000:2.0:1767320040.669324:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324544 00000100:00100000:2.0:1767320040.669355:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3251 00000100:00100000:2.0:1767320040.669362:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985324416:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320040.669416:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985324416:12345-0@lo:400:kworker.0 Request processed in 54us (1508us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320040.669422:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3251 00000100:00100000:2.0:1767320040.669436:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3252 00000100:00100000:2.0:1767320040.669443:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985324544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320040.669468:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985324544:12345-0@lo:400:kworker.0 Request processed in 27us (1466us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320040.669474:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3252 00000100:00100000:2.0:1767320040.669503:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985323520:0@lo:250:kworker.0 00000400:00000100:2.0:1767320040.669537:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985323520 offset 0 length 520: 4 00000100:00100000:2.0:1767320040.669560:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985323648:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.669592:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.669611:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985323776:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.669634:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.669650:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985324032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.669681:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.669697:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985324288:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.669714:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.669744:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985323776 00000100:00100000:2.0:1767320040.669748:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985323776 00000100:00100000:2.0:1767320040.669768:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1886 00000100:00100000:2.0:1767320040.669772:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985323776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320040.669788:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320040.669817:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985323776:12345-0@lo:38:kworker.0 Request processed in 43us (182us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320040.669823:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1886 00000100:00100000:2.0:1767320040.669839:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985323648 00000100:00100000:2.0:1767320040.669842:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985323648 00000100:00100000:2.0:1767320040.669875:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324032 00000100:00100000:2.0:1767320040.669878:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324032 00000100:00100000:2.0:1767320040.669889:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324288 00000100:00100000:2.0:1767320040.669891:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324288 00000100:00100000:2.0:1767320040.669905:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5582 00000100:00100000:2.0:1767320040.669908:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985323648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320040.669917:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320040.669938:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985323648:12345-0@lo:38:kworker.0 Request processed in 29us (346us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320040.669953:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5582 00000100:00100000:2.0:1767320040.669965:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5583 00000100:00100000:2.0:1767320040.669968:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985324032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320040.669977:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320040.670000:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985324032:12345-0@lo:38:kworker.0 Request processed in 30us (319us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320040.670005:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5583 00000100:00100000:2.0:1767320040.670014:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5584 00000100:00100000:2.0:1767320040.670018:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985324288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320040.670025:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320040.670041:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985324288:12345-0@lo:38:kworker.0 Request processed in 22us (326us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320040.670046:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5584 00000100:00100000:2.0:1767320040.670088:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985324544:0@lo:400:kworker.0 00000100:00100000:2.0:1767320040.670099:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985324416:0@lo:400:kworker.0 00000100:00080000:2.0:1767320040.670128:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.670133:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.670136:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985323648:0@lo:38:kworker.0 00000100:00080000:2.0:1767320040.670143:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.670146:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.670148:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985323776:0@lo:38:kworker.0 00000100:00080000:2.0:1767320040.670154:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.670157:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.670160:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985324032:0@lo:38:kworker.0 00000100:00080000:2.0:1767320040.670194:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.670198:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.670200:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985324288:0@lo:38:kworker.0 00000100:00080000:1.0:1767320040.670360:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320040.670364:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320041.312026:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320042.203385:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320042.203460:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985324672:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320042.203529:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320042.203853:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985324800:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320042.203883:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320042.204911:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324672 00000100:00100000:0.0:1767320042.204922:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324672 00000100:00100000:0.0:1767320042.205020:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324800 00000100:00100000:0.0:1767320042.205024:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324800 00000100:00080000:1.0:1767320042.205064:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320042.205069:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320042.205102:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2440 00000100:00100000:2.0:1767320042.205113:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985324672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320042.205141:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320042.205257:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985324672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 145us (1726us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320042.205265:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2440 00000100:00100000:2.0:1767320042.205280:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2441 00000100:00100000:2.0:1767320042.205285:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985324800:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320042.205300:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320042.205321:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985324800:12345-0@lo:13:osp-pre-0-1.0 Request processed in 37us (1438us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320042.205327:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2441 00000100:00100000:1.0:1767320042.205554:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985324672:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320042.205589:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985324800:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320042.332534:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320043.365192:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320044.380202:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320045.405630:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320045.747509:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320045.747573:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320045.747667:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920595072 00000100:00100000:2.0:1767320045.747677:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920595072 00000100:00100000:2.0:1767320045.747747:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920594944 00000100:00100000:2.0:1767320045.747751:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920594944 00000100:00100000:2.0:1767320045.747772:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4985 00000100:00100000:2.0:1767320045.747785:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920595072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767320045.747805:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320045.747842:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5585 00000100:00100000:0.0:1767320045.747850:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920594944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320045.747872:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320045.747881:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920595072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 97us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320045.747889:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4985 00000100:00100000:0.0:1767320045.747917:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920594944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 65us (342us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320045.747925:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5585 00000100:00080000:1.0:1767320045.787534:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320045.787545:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320045.787548:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320045.787552:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320045.787557:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320045.787565:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5288 00000100:00080000:1.0:1767320045.787569:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320045.787575:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320045.787614:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4a80 x1853163985324928/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320045.787643:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320045.787646:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320045.787648:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320045.787651:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5288 00000100:00080000:1.0:1767320045.787653:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320045.787657:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320045.787673:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985325056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320045.787715:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320045.787719:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320045.787721:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320045.787725:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5288 00000100:00080000:1.0:1767320045.787727:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320045.787730:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320045.787741:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4000 x1853163985325312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320045.787758:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320045.787760:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320045.787762:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320045.787777:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5288 00000100:00080000:1.0:1767320045.787780:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320045.787783:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320045.788080:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985325184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.788146:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320045.788224:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320045.788229:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985325440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.788249:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320045.788280:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325184 00000100:00100000:3.0:1767320045.788287:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325184 00000100:00100000:3.0:1767320045.788336:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325440 00000100:00100000:3.0:1767320045.788339:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325440 00000100:00100000:3.0:1767320045.788359:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4986 00000100:00100000:3.0:1767320045.788368:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985325184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.788419:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985325184:12345-0@lo:400:kworker.0 Request processed in 53us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320045.788426:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4986 00000100:00100000:3.0:1767320045.788442:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4987 00000100:00100000:3.0:1767320045.788447:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985325440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.788470:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985325440:12345-0@lo:400:kworker.0 Request processed in 23us (221us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320045.788475:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4987 00000100:00080000:3.0:1767320045.788504:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320045.788532:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985325440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.788541:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985325184:0@lo:400:kworker.0 00000100:00100000:1.0:1767320045.788816:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5c00 x1853163985325568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320045.788972:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985325696:0@lo:400:kworker.0 00000100:00100000:0.0:1767320045.789008:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320045.789041:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325696 00000100:00100000:0.0:1767320045.789045:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325696 00000100:00100000:0.0:1767320045.789080:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3253 00000100:00100000:0.0:1767320045.789085:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985325696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320045.789115:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985325696:12345-0@lo:400:kworker.0 Request processed in 30us (106us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320045.789120:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3253 00000100:00080000:0.0:1767320045.789137:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320045.789151:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985325696:0@lo:400:kworker.0 00000100:00100000:1.0:1767320045.789274:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985325824:0@lo:400:kworker.0 00000100:00100000:1.0:1767320045.789302:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320045.789371:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325824 00000100:00100000:0.0:1767320045.789374:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325824 00000100:00100000:0.0:1767320045.789410:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3254 00000100:00100000:0.0:1767320045.789416:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985325824:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320045.789450:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985325824:12345-0@lo:400:kworker.0 Request processed in 34us (147us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320045.789456:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3254 00000100:00100000:0.0:1767320045.789489:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985325824:0@lo:400:kworker.0 00000100:00080000:1.0:1767320045.789660:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320045.789719:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985324928:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.789751:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320045.789939:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985325056:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.789963:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320045.789991:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985325312:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.790011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320045.790056:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325056 00000100:00100000:0.0:1767320045.790062:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325056 00000100:00100000:0.0:1767320045.790092:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1887 00000100:00100000:0.0:1767320045.790097:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985325056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320045.790114:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320045.790143:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985325056:12345-0@lo:38:kworker.0 Request processed in 45us (179us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320045.790149:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1887 00000100:00100000:2.0:1767320045.791784:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985325568:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.791834:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320045.792415:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985324928 00000100:00100000:3.0:1767320045.792427:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985324928 00000100:00100000:3.0:1767320045.792472:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325312 00000100:00100000:3.0:1767320045.792475:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325312 00000100:00100000:3.0:1767320045.792509:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5586 00000100:00100000:3.0:1767320045.792518:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985324928:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320045.792542:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320045.792598:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985324928:12345-0@lo:38:kworker.0 Request processed in 80us (2845us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320045.792606:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5586 00000100:00100000:3.0:1767320045.792621:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5587 00000100:00100000:3.0:1767320045.792625:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985325312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320045.792633:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320045.792657:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985325312:12345-0@lo:38:kworker.0 Request processed in 30us (2644us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320045.792662:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5587 00000100:00080000:2.0:1767320045.793194:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.793201:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.793205:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985324928:0@lo:38:kworker.0 00000100:00080000:2.0:1767320045.793219:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.793222:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.793225:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985325056:0@lo:38:kworker.0 00000100:00080000:2.0:1767320045.793234:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.793237:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.793240:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985325312:0@lo:38:kworker.0 00000100:00100000:1.0:1767320045.793317:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325568 00000100:00100000:1.0:1767320045.793323:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325568 00000100:00100000:0.0:1767320045.793378:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5588 00000100:00100000:0.0:1767320045.793385:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985325568:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320045.793397:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320045.793496:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985325568:12345-0@lo:38:kworker.0 Request processed in 110us (1661us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320045.793503:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5588 00000100:00080000:2.0:1767320045.793632:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.793636:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.793639:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985325568:0@lo:38:kworker.0 00010000:00010000:2.0:1767320046.430632:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320047.325083:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985325952:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320047.325157:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320047.325608:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985325952 00000100:00100000:3.0:1767320047.325616:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985325952 00000100:00100000:3.0:1767320047.325689:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2442 00000100:00100000:3.0:1767320047.325699:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985325952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320047.325728:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320047.325774:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985325952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (617us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320047.325781:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2442 00000100:00100000:3.0:1767320047.325823:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985325952:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320047.325906:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320047.328069:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985326080:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320047.328129:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320047.328314:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326080 00000100:00100000:2.0:1767320047.328323:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326080 00000100:00100000:2.0:1767320047.328383:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2443 00000100:00100000:2.0:1767320047.328392:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985326080:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320047.328420:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320047.328472:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985326080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (343us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320047.328480:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2443 00000100:00080000:1.0:1767320047.328544:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320047.328850:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985326080:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320047.451569:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320048.476227:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320049.500819:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320050.526062:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320050.862934:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320050.862987:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320050.863322:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920595328 00000100:00100000:1.0:1767320050.863331:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920595328 00000100:00100000:1.0:1767320050.863625:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920595200 00000100:00100000:1.0:1767320050.863629:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920595200 00000100:00100000:1.0:1767320050.863648:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4988 00000100:00100000:1.0:1767320050.863657:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920595328:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320050.863729:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920595328:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (806us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320050.863737:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4988 00000100:00100000:1.0:1767320050.863753:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5589 00000100:00100000:1.0:1767320050.863758:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920595200:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320050.863778:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320050.863799:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920595200:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (812us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320050.863806:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5589 00000100:00080000:0.0:1767320050.864438:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320050.907408:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320050.907418:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320050.907422:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320050.907426:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320050.907430:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320050.907438:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5294 00000100:00080000:1.0:1767320050.907442:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320050.907449:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320050.907486:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5180 x1853163985326208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320050.907514:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320050.907518:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320050.907520:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320050.907524:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5294 00000100:00080000:1.0:1767320050.907526:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320050.907530:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320050.907542:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5880 x1853163985326336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320050.907586:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320050.907590:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320050.907592:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320050.907596:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5294 00000100:00080000:1.0:1767320050.907598:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320050.907603:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320050.907613:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7480 x1853163985326592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320050.907629:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320050.907632:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320050.907634:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320050.907637:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5294 00000100:00080000:1.0:1767320050.907640:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320050.907643:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320050.907651:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5c00 x1853163985326848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320050.907728:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985326208:0@lo:38:kworker.0 00000100:00100000:2.0:1767320050.907790:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320050.907845:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985326336:0@lo:38:kworker.0 00000100:00100000:2.0:1767320050.907869:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320050.907892:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985326592:0@lo:38:kworker.0 00000100:00100000:2.0:1767320050.907910:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320050.907922:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985326848:0@lo:38:kworker.0 00000100:00100000:2.0:1767320050.907939:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320050.907966:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326336 00000100:00100000:2.0:1767320050.907974:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326336 00000100:00100000:2.0:1767320050.908005:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1888 00000100:00100000:2.0:1767320050.908012:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985326336:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320050.908030:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320050.908064:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985326336:12345-0@lo:38:kworker.0 Request processed in 52us (194us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320050.908072:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1888 00000100:00080000:2.0:1767320050.908109:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320050.908113:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320050.908117:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985326336:0@lo:38:kworker.0 00000100:00100000:1.0:1767320050.908308:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326208 00000100:00100000:1.0:1767320050.908312:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326208 00000100:00100000:1.0:1767320050.908339:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326592 00000100:00100000:1.0:1767320050.908342:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326592 00000100:00100000:1.0:1767320050.908353:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326848 00000100:00100000:1.0:1767320050.908355:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326848 00000100:00100000:1.0:1767320050.908367:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5590 00000100:00100000:1.0:1767320050.908372:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985326208:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320050.908768:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985326720:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.908801:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320050.908819:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320050.908824:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985326464:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.908842:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320050.908863:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326720 00000100:00100000:3.0:1767320050.908866:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326720 00000100:00100000:3.0:1767320050.908899:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326464 00000100:00100000:3.0:1767320050.908902:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326464 00000100:00100000:3.0:1767320050.908916:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4989 00000100:00100000:3.0:1767320050.908921:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985326720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.908960:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985326720:12345-0@lo:400:kworker.0 Request processed in 39us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320050.908966:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4989 00000100:00100000:3.0:1767320050.908980:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4990 00000100:00100000:3.0:1767320050.908986:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985326464:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.909010:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985326464:12345-0@lo:400:kworker.0 Request processed in 25us (167us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320050.909016:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4990 00000100:00100000:3.0:1767320050.909026:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5591 00000100:00100000:3.0:1767320050.909030:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985326592:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320050.909040:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320050.909065:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985326592:12345-0@lo:38:kworker.0 Request processed in 34us (1154us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320050.909070:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5591 00000100:00100000:3.0:1767320050.909081:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5592 00000100:00100000:3.0:1767320050.909085:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985326848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320050.909094:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320050.909116:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985326848:12345-0@lo:38:kworker.0 Request processed in 29us (1176us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320050.909121:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5592 00000100:00100000:3.0:1767320050.909147:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985326976:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.909198:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320050.909218:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320050.909236:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985327104:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.909254:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320050.909286:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985326464:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.909295:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985326720:0@lo:400:kworker.0 00000100:00080000:3.0:1767320050.909313:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320050.909689:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320050.909696:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320050.909699:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985326592:0@lo:38:kworker.0 00000100:00080000:2.0:1767320050.909708:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320050.909711:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320050.909714:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985326848:0@lo:38:kworker.0 00010000:00020000:1.0:1767320050.909876:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320050.909925:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985326208:12345-0@lo:38:kworker.0 Request processed in 1552us (2135us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320050.909932:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5590 00000100:00100000:0.0:1767320050.910256:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985326976 00000100:00100000:0.0:1767320050.910262:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985326976 00000100:00100000:0.0:1767320050.910312:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985327104 00000100:00100000:0.0:1767320050.910315:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985327104 00000100:00100000:0.0:1767320050.910338:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3255 00000100:00100000:0.0:1767320050.910346:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985326976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320050.910397:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985326976:12345-0@lo:400:kworker.0 Request processed in 52us (1198us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320050.910405:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3255 00000100:00100000:0.0:1767320050.910417:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3256 00000100:00100000:0.0:1767320050.910424:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985327104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320050.910448:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985327104:12345-0@lo:400:kworker.0 Request processed in 25us (1193us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320050.910454:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3256 00000100:00100000:0.0:1767320050.910488:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985327104:0@lo:400:kworker.0 00000100:00100000:0.0:1767320050.910498:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985326976:0@lo:400:kworker.0 00000100:00080000:2.0:1767320050.910644:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320050.910649:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320050.910654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985326208:0@lo:38:kworker.0 00000100:00080000:3.0:1767320050.911422:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320050.911427:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320051.548880:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320052.443728:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985327232:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320052.443799:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320052.443984:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320052.443993:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985327360:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320052.444019:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320052.444089:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985327232 00000100:00100000:1.0:1767320052.444097:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985327232 00000100:00100000:1.0:1767320052.444154:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985327360 00000100:00100000:1.0:1767320052.444157:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985327360 00000100:00100000:1.0:1767320052.444210:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2444 00000100:00100000:1.0:1767320052.444219:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985327232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320052.444247:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320052.444289:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985327232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (489us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320052.444296:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2444 00000100:00100000:1.0:1767320052.444312:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2445 00000100:00100000:1.0:1767320052.444318:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985327360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320052.444331:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320052.444366:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985327360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 37us (335us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320052.444373:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2445 00000100:00100000:0.0:1767320052.444475:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985327360:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320052.444487:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985327232:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320052.444510:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320052.444513:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320052.573018:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320053.595932:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320054.619435:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320055.643406:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320055.982572:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320055.982697:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320055.991797:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920595456 00000100:00100000:1.0:1767320055.991814:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920595456 00000100:00100000:1.0:1767320055.991879:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920595584 00000100:00100000:1.0:1767320055.991883:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920595584 00000100:00100000:1.0:1767320055.992001:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4991 00000100:00100000:1.0:1767320055.992014:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920595584:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320055.992096:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920595584:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (9399us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320055.992103:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4991 00000100:00100000:1.0:1767320055.992121:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5593 00000100:00100000:1.0:1767320055.992126:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920595456:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320055.992157:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320055.992200:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920595456:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 72us (9640us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320055.992206:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5593 00000100:00080000:0.0:1767320055.992847:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320056.029580:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320056.029595:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320056.029689:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320056.029694:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320056.029700:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320056.029711:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5299 00000100:00080000:3.0:1767320056.029717:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320056.029726:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320056.029782:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7c380 x1853163985327488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320056.029885:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985327488:0@lo:38:kworker.0 00000100:00100000:2.0:1767320056.029999:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320056.030196:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985327488 00000100:00100000:0.0:1767320056.030208:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985327488 00000100:00100000:0.0:1767320056.030258:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5594 00000100:00100000:0.0:1767320056.030268:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985327488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320056.030307:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320056.030394:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320056.030398:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320056.030400:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320056.030405:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5299 00000100:00080000:3.0:1767320056.030408:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320056.030412:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320056.030430:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7e680 x1853163985327616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320056.032336:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985327616:0@lo:38:kworker.0 00000100:00100000:2.0:1767320056.032416:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320056.032674:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985327616 00000100:00100000:1.0:1767320056.032689:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985327616 00000100:00100000:1.0:1767320056.032742:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1889 00000100:00100000:1.0:1767320056.032750:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985327616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320056.032772:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320056.032848:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985327616:12345-0@lo:38:kworker.0 Request processed in 97us (431us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320056.032857:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1889 00000100:00100000:0.0:1767320056.032876:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985327488:12345-0@lo:38:kworker.0 Request processed in 2602us (2871us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320056.032888:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5594 00000100:00080000:3.0:1767320056.033248:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320056.033254:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320056.033257:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320056.033263:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5299 00000100:00080000:3.0:1767320056.033268:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320056.033273:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320056.033304:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7c000 x1853163985327872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320056.033334:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320056.033337:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320056.033339:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320056.033343:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5299 00000100:00080000:3.0:1767320056.033345:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320056.033348:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320056.033358:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7df80 x1853163985328128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320056.033414:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985328256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320056.033454:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320056.033478:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320056.033483:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985328384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320056.033503:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320056.033688:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985328000:0@lo:400:kworker.0 00000100:00100000:1.0:1767320056.033738:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320056.033786:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328256 00000100:00100000:1.0:1767320056.033786:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320056.033797:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328256 00000100:00100000:1.0:1767320056.033802:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985327744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320056.033834:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320056.033873:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328384 00000100:00100000:0.0:1767320056.033879:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328384 00000100:00100000:1.0:1767320056.033897:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3257 00000100:00100000:1.0:1767320056.033922:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985328256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320056.033970:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985328256:12345-0@lo:400:kworker.0 Request processed in 53us (514us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320056.033977:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3257 00000100:00100000:1.0:1767320056.034005:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328000 00000100:00100000:1.0:1767320056.034010:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328000 00000100:00100000:1.0:1767320056.034043:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985327744 00000100:00100000:1.0:1767320056.034046:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985327744 00000100:00100000:1.0:1767320056.034067:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4992 00000100:00100000:1.0:1767320056.034073:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985328000:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320056.034102:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985328000:12345-0@lo:400:kworker.0 Request processed in 30us (364us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320056.034107:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4992 00000100:00100000:1.0:1767320056.034117:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4993 00000100:00100000:1.0:1767320056.034123:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985327744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320056.034160:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985327744:12345-0@lo:400:kworker.0 Request processed in 37us (324us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320056.034197:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4993 00000100:00100000:1.0:1767320056.034511:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985327744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320056.034596:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985328000:0@lo:400:kworker.0 00000100:00080000:1.0:1767320056.034648:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320056.034653:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320056.034655:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320056.036158:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3258 00000100:00100000:0.0:1767320056.036198:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985328384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320056.036308:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985328384:12345-0@lo:400:kworker.0 Request processed in 124us (2802us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320056.036317:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3258 00000100:00100000:3.0:1767320056.036741:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985328384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320056.036760:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985328256:0@lo:400:kworker.0 00000100:00080000:2.0:1767320056.037107:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320056.037114:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320056.037118:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985327488:0@lo:38:kworker.0 00000100:00100000:2.0:1767320056.037143:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985327872:0@lo:38:kworker.0 00000100:00100000:2.0:1767320056.037207:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320056.037281:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985328128:0@lo:38:kworker.0 00000100:00100000:2.0:1767320056.037315:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320056.037502:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320056.037507:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320056.037510:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985327616:0@lo:38:kworker.0 00000100:00100000:3.0:1767320056.037568:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985327872 00000100:00100000:3.0:1767320056.037576:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985327872 00000100:00100000:0.0:1767320056.037767:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328128 00000100:00100000:0.0:1767320056.037772:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328128 00000100:00100000:0.0:1767320056.037848:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5595 00000100:00100000:0.0:1767320056.037855:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985327872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320056.037879:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320056.037954:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985327872:12345-0@lo:38:kworker.0 Request processed in 98us (747us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320056.037962:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5595 00000100:00100000:0.0:1767320056.037978:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5596 00000100:00100000:0.0:1767320056.037982:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985328128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320056.037991:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320056.038014:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985328128:12345-0@lo:38:kworker.0 Request processed in 31us (699us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320056.038019:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5596 00000100:00080000:2.0:1767320056.038246:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320056.038251:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320056.038255:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985327872:0@lo:38:kworker.0 00000100:00080000:2.0:1767320056.038263:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320056.038265:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320056.038269:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985328128:0@lo:38:kworker.0 00010000:00010000:0.0:1767320056.669077:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320057.564330:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320057.564359:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985328512:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320057.564454:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320057.564877:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328512 00000100:00100000:3.0:1767320057.564888:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328512 00000100:00100000:3.0:1767320057.564955:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2446 00000100:00100000:3.0:1767320057.564967:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985328512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320057.565005:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320057.565064:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985328512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (609us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320057.565073:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2446 00000100:00100000:3.0:1767320057.565129:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985328512:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320057.569760:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985328640:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320057.569853:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320057.570215:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328640 00000100:00100000:2.0:1767320057.570225:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328640 00000100:00100000:2.0:1767320057.570302:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2447 00000100:00100000:2.0:1767320057.570314:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985328640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320057.570350:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320057.570413:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985328640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (560us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320057.570421:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2447 00000100:00100000:0.0:1767320057.570733:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985328640:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320057.692110:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320058.718002:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320059.739443:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320060.766158:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320061.098799:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320061.098892:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320061.098936:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920595840 00000100:00100000:2.0:1767320061.098952:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920595840 00000100:00100000:2.0:1767320061.099039:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920595712 00000100:00100000:2.0:1767320061.099043:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920595712 00000100:00100000:2.0:1767320061.099069:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4994 00000100:00100000:2.0:1767320061.099081:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920595840:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320061.099217:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920595840:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320061.099226:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4994 00000100:00100000:2.0:1767320061.099246:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5597 00000100:00100000:2.0:1767320061.099251:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920595712:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320061.099271:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320061.099292:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320061.099318:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920595712:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (413us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320061.099327:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5597 00000100:00080000:3.0:1767320061.147377:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320061.147392:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320061.147395:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320061.147400:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320061.147406:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320061.147417:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5304 00000100:00080000:3.0:1767320061.147422:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320061.147431:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320061.147500:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579180 x1853163985328768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320061.147536:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320061.147541:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320061.147543:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320061.147548:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5304 00000100:00080000:3.0:1767320061.147550:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320061.147555:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320061.147573:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579880 x1853163985328896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320061.147621:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320061.147626:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320061.147628:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320061.147632:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5304 00000100:00080000:3.0:1767320061.147635:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320061.147638:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320061.147652:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b800 x1853163985329152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320061.147670:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320061.147673:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320061.147675:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320061.147679:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5304 00000100:00080000:3.0:1767320061.147681:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320061.147685:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320061.147697:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357aa00 x1853163985329408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320061.147778:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985329280:0@lo:400:kworker.0 00000100:00100000:2.0:1767320061.147800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985328768:0@lo:38:kworker.0 00000100:00100000:3.0:1767320061.147877:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320061.148009:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985329024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320061.148284:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985329664:0@lo:400:kworker.0 00000100:00100000:2.0:1767320061.157117:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320061.157232:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985328896:0@lo:38:kworker.0 00000100:00100000:2.0:1767320061.157296:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320061.157816:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329280 00000100:00100000:3.0:1767320061.157834:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329280 00000100:00100000:3.0:1767320061.157908:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328768 00000100:00100000:3.0:1767320061.157912:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328768 00000100:00100000:3.0:1767320061.157941:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4995 00000100:00100000:3.0:1767320061.157952:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985329280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320061.158059:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985329280:12345-0@lo:400:kworker.0 Request processed in 109us (10184us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320061.158068:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4995 00000100:00100000:3.0:1767320061.158091:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5598 00000100:00100000:3.0:1767320061.158095:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985328768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320061.158121:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320061.158190:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985328768:12345-0@lo:38:kworker.0 Request processed in 93us (1081us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320061.158205:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5598 00000100:00100000:3.0:1767320061.158268:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985329280:0@lo:400:kworker.0 00000100:00080000:3.0:1767320061.158311:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320061.158314:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985328896 00000100:00100000:2.0:1767320061.158326:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985328896 00000100:00100000:2.0:1767320061.158375:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1890 00000100:00100000:2.0:1767320061.158382:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985328896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320061.158406:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320061.158455:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985328896:12345-0@lo:38:kworker.0 Request processed in 72us (1158us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320061.158464:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1890 00000100:00080000:3.0:1767320061.158512:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320061.158521:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320061.158525:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985328896:0@lo:38:kworker.0 00000100:00100000:3.0:1767320061.158536:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985329152:0@lo:38:kworker.0 00000100:00100000:3.0:1767320061.158582:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320061.158603:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985329408:0@lo:38:kworker.0 00000100:00100000:3.0:1767320061.158624:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320061.158644:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329152 00000100:00100000:3.0:1767320061.158648:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329152 00000100:00100000:3.0:1767320061.158666:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329408 00000100:00100000:3.0:1767320061.158668:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329408 00000100:00100000:3.0:1767320061.158682:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5599 00000100:00100000:3.0:1767320061.158687:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985329152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320061.158699:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320061.158728:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985329152:12345-0@lo:38:kworker.0 Request processed in 40us (146us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320061.158736:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5599 00000100:00100000:3.0:1767320061.158755:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5600 00000100:00100000:3.0:1767320061.158760:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985329408:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320061.158770:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320061.158797:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985329408:12345-0@lo:38:kworker.0 Request processed in 36us (172us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320061.158804:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5600 00000100:00080000:3.0:1767320061.158839:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320061.158843:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320061.158847:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985329408:0@lo:38:kworker.0 00000100:00080000:3.0:1767320061.158878:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320061.158884:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320061.158889:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985328768:0@lo:38:kworker.0 00000100:00080000:3.0:1767320061.158899:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320061.158903:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320061.158906:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985329152:0@lo:38:kworker.0 00000100:00100000:3.0:1767320061.158957:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985329536:0@lo:400:kworker.0 00000100:00100000:3.0:1767320061.158997:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320061.159055:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329536 00000100:00100000:2.0:1767320061.159064:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329536 00000100:00100000:2.0:1767320061.159127:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3259 00000100:00100000:2.0:1767320061.159135:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985329536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320061.159408:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985329536:12345-0@lo:400:kworker.0 Request processed in 273us (411us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320061.159416:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3259 00000100:00100000:3.0:1767320061.164601:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985329536:0@lo:400:kworker.0 00000100:00080000:3.0:1767320061.164652:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320061.167278:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320061.168801:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329024 00000100:00100000:3.0:1767320061.168813:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329024 00000100:00100000:3.0:1767320061.168886:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4996 00000100:00100000:3.0:1767320061.168896:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985329024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320061.168963:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985329024:12345-0@lo:400:kworker.0 Request processed in 69us (1694us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320061.168970:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4996 00000100:00080000:0.0:1767320061.169128:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320061.169160:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985329024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320061.176024:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320061.176781:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329664 00000100:00100000:0.0:1767320061.176796:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329664 00000100:00100000:0.0:1767320061.176886:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3260 00000100:00100000:0.0:1767320061.176899:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985329664:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320061.177012:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985329664:12345-0@lo:400:kworker.0 Request processed in 116us (997us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320061.177021:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3260 00000100:00100000:1.0:1767320061.177296:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985329664:0@lo:400:kworker.0 00000100:00080000:3.0:1767320061.178152:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767320061.787411:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320062.687541:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320062.687566:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985329792:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320062.687625:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320062.688442:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329792 00000100:00100000:0.0:1767320062.688458:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329792 00000100:00100000:0.0:1767320062.688552:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2448 00000100:00100000:0.0:1767320062.688563:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985329792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320062.688619:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320062.688719:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985329792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 156us (1089us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320062.688728:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2448 00000100:00100000:3.0:1767320062.688764:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985329792:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320062.689797:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985329920:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320062.689851:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320062.690049:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985329920 00000100:00100000:1.0:1767320062.690059:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985329920 00000100:00100000:1.0:1767320062.690110:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2449 00000100:00100000:1.0:1767320062.690120:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985329920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320062.690158:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320062.690284:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985329920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 165us (431us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320062.690292:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2449 00000100:00100000:3.0:1767320062.690364:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985329920:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320062.813220:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320063.835623:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320064.859918:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320065.243317:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320040/real 1767320040] req@ffff8ce188631180 x1853163985323520/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320065 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320065.243351:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188631180 x1853163985323520/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320065 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320065.243371:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320065.243395:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320065.243398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985323520:0@lo:250:kworker.0 00000100:00080000:3.0:1767320065.243902:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.243908:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320065.243911:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.243916:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5288 00000100:00080000:3.0:1767320065.243921:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.243927:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320065.244392:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357aa00 x1853163985330048/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.244498:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.244502:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.244504:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.244508:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.244511:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.244515:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320065.244656:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579880 x1853163985330176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.244665:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.244669:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.244670:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.244674:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.244676:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.244679:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320065.244692:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578e00 x1853163985330304/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.244731:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.244735:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.244737:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.244740:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.244742:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.244745:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320065.244758:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985330560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.244776:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.244779:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.244781:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.244785:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.244788:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.244794:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320065.244806:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b100 x1853163985330816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320065.244818:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985330432:0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.244876:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985330944:0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.244928:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320065.245011:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985330048:0@lo:250:kworker.0 00000100:00100000:2.0:1767320065.245022:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985330432 00000100:00100000:2.0:1767320065.245035:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985330432 00000100:00100000:3.0:1767320065.245063:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320065.245138:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4997 00000100:00100000:3.0:1767320065.245143:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320065.245156:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985330432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.245160:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985331072:0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.245203:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320065.245214:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985330688:0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.245247:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320065.245261:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985330432:12345-0@lo:400:kworker.0 Request processed in 112us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320065.245269:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4997 00000100:00100000:1.0:1767320065.245285:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985330432:0@lo:400:kworker.0 00000100:00100000:2.0:1767320065.245290:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985330688 00000100:00100000:2.0:1767320065.245294:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985330688 00000100:00100000:1.0:1767320065.245342:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4998 00000100:00100000:1.0:1767320065.245352:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985330688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.245395:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985330688:12345-0@lo:400:kworker.0 Request processed in 45us (147us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320065.245404:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4998 00000100:00100000:1.0:1767320065.245430:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985330944 00000100:00100000:1.0:1767320065.245445:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985330944 00000100:00100000:1.0:1767320065.245480:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3261 00000100:00100000:1.0:1767320065.245490:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985330944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.245505:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320065.245541:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985330944:12345-0@lo:400:kworker.0 Request processed in 52us (478us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320065.245562:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3261 00000100:00100000:1.0:1767320065.245574:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331072 00000100:00100000:1.0:1767320065.245578:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331072 00000100:00100000:1.0:1767320065.245607:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3262 00000100:00100000:1.0:1767320065.245613:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985331072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.245642:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985331072:12345-0@lo:400:kworker.0 Request processed in 30us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320065.245649:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3262 00000100:00080000:1.0:1767320065.245673:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320065.245680:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320065.245683:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320065.245698:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985331072:0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.245727:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985330944:0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.245735:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985330688:0@lo:400:kworker.0 00000400:00000100:0.0:1767320065.246276:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985330048 offset 0 length 520: 4 00000100:00100000:0.0:1767320065.246791:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985330176:0@lo:38:kworker.0 00000100:00100000:0.0:1767320065.246830:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320065.246860:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985330304:0@lo:38:kworker.0 00000100:00100000:0.0:1767320065.246885:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320065.246908:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985330560:0@lo:38:kworker.0 00000100:00100000:0.0:1767320065.246930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320065.246955:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985330816:0@lo:38:kworker.0 00000100:00100000:0.0:1767320065.246980:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320065.247008:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985330176 00000100:00100000:1.0:1767320065.247020:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985330176 00000100:00100000:3.0:1767320065.247084:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985330560 00000100:00100000:3.0:1767320065.247092:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985330560 00000100:00100000:1.0:1767320065.247191:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985330816 00000100:00100000:1.0:1767320065.247196:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985330816 00000100:00100000:2.0:1767320065.247228:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985330304 00000100:00100000:2.0:1767320065.247233:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985330304 00000100:00100000:2.0:1767320065.247263:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1891 00000100:00100000:2.0:1767320065.247270:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985330304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320065.247294:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320065.247716:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5601 00000100:00100000:3.0:1767320065.247726:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985330560:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320065.247734:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5602 00000100:00100000:1.0:1767320065.247745:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985330176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320065.247754:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320065.247762:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320065.248384:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985330560:12345-0@lo:38:kworker.0 Request processed in 655us (1449us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320065.248394:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5601 00000100:00100000:1.0:1767320065.248404:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985330176:12345-0@lo:38:kworker.0 Request processed in 651us (1565us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320065.248416:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5602 00000100:00100000:1.0:1767320065.248443:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5603 00000100:00100000:1.0:1767320065.248449:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985330816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320065.248461:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320065.248500:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985330816:12345-0@lo:38:kworker.0 Request processed in 49us (1519us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320065.248507:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5603 00000100:00080000:1.0:1767320065.248592:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320065.248601:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320065.248606:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985330176:0@lo:38:kworker.0 00000100:00080000:1.0:1767320065.248620:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320065.248623:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320065.248626:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985330560:0@lo:38:kworker.0 00000100:00080000:1.0:1767320065.248636:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320065.248639:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320065.248642:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985330816:0@lo:38:kworker.0 00000100:00100000:2.0:1767320065.249762:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985330304:12345-0@lo:38:kworker.0 Request processed in 2485us (2869us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320065.249775:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1891 00000100:00080000:1.0:1767320065.250005:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320065.250010:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320065.250013:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985330304:0@lo:38:kworker.0 00010000:00010000:1.0:1767320065.883388:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320066.219897:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320066.220126:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596224 00000100:00100000:0.0:1767320066.220138:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596224 00000100:00100000:0.0:1767320066.220247:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 4999 00000100:00100000:0.0:1767320066.220257:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920596224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320066.220338:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920596224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (451us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320066.220346:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 4999 00000100:00080000:1.0:1767320066.221222:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:1.0:1767320066.228588:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920595968 offset 0 length 520: 4 00000100:00100000:1.0:1767320066.228640:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320066.229136:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596096 00000100:00100000:3.0:1767320066.229147:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596096 00000100:00100000:3.0:1767320066.229235:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5604 00000100:00100000:3.0:1767320066.229243:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920596096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320066.229267:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320066.229340:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920596096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 96us (699us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320066.229349:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5604 00010000:00010000:2.0:1767320066.909358:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320067.803860:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985331200:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320067.803941:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320067.804117:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331200 00000100:00100000:2.0:1767320067.804126:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331200 00000100:00100000:2.0:1767320067.804222:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2450 00000100:00100000:2.0:1767320067.804232:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985331200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320067.804262:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320067.804307:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985331200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (365us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320067.804315:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2450 00000100:00080000:3.0:1767320067.804383:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320067.804421:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985331200:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320067.804784:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320067.804792:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985331328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320067.804822:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320067.805045:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331328 00000100:00100000:1.0:1767320067.805050:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331328 00000100:00100000:1.0:1767320067.805101:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2451 00000100:00100000:1.0:1767320067.805108:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985331328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320067.805125:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320067.805182:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985331328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320067.805188:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2451 00000100:00100000:1.0:1767320067.805223:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985331328:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320067.805456:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767320067.931340:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320068.956449:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320069.979681:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320070.363317:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320070.363328:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320070.363332:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320070.363336:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320070.363341:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320070.363349:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5313 00000100:00080000:3.0:1767320070.363354:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320070.363360:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320070.363400:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7100 x1853163985331456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320070.363426:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320070.363430:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320070.363432:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320070.363443:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5313 00000100:00080000:3.0:1767320070.363446:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320070.363450:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320070.363463:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4380 x1853163985331584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320070.363519:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320070.363524:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320070.363526:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320070.363530:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5313 00000100:00080000:3.0:1767320070.363533:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320070.363536:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320070.363550:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6a00 x1853163985331840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320070.363565:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320070.363576:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320070.363578:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320070.363581:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5313 00000100:00080000:3.0:1767320070.363583:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320070.363586:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320070.363599:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5180 x1853163985332096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320070.363655:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985332224:0@lo:400:kworker.0 00000100:00100000:3.0:1767320070.363717:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320070.363752:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985331712:0@lo:400:kworker.0 00000100:00100000:3.0:1767320070.363779:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320070.363784:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320070.363786:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985332352:0@lo:400:kworker.0 00000100:00100000:3.0:1767320070.363805:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320070.363811:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320070.363816:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985331968:0@lo:400:kworker.0 00000100:00100000:3.0:1767320070.363832:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332224 00000100:00100000:0.0:1767320070.363834:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320070.363838:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332224 00000100:00100000:0.0:1767320070.363858:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331712 00000100:00100000:0.0:1767320070.363874:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331712 00000100:00100000:3.0:1767320070.363881:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332352 00000100:00100000:3.0:1767320070.363883:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332352 00000100:00100000:0.0:1767320070.363897:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331968 00000100:00100000:0.0:1767320070.363899:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331968 00000100:00100000:3.0:1767320070.363900:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3263 00000100:00100000:3.0:1767320070.363907:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985332224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320070.363915:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5000 00000100:00100000:0.0:1767320070.363919:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985331712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320070.363952:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985331712:12345-0@lo:400:kworker.0 Request processed in 30us (165us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320070.363952:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985332224:12345-0@lo:400:kworker.0 Request processed in 44us (234us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320070.363958:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5000 00000100:00100000:3.0:1767320070.363959:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3263 00000100:00100000:0.0:1767320070.363974:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5001 00000100:00100000:3.0:1767320070.363974:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3264 00000100:00100000:0.0:1767320070.363980:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985331968:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320070.363980:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985332352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320070.364003:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985331968:12345-0@lo:400:kworker.0 Request processed in 24us (168us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320070.364003:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985332352:12345-0@lo:400:kworker.0 Request processed in 24us (197us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320070.364009:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5001 00000100:00100000:3.0:1767320070.364009:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3264 00000100:00080000:3.0:1767320070.364035:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320070.364039:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320070.364041:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320070.364056:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985331968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320070.364064:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985331712:0@lo:400:kworker.0 00000100:00100000:3.0:1767320070.364064:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985332352:0@lo:400:kworker.0 00000100:00100000:3.0:1767320070.364072:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985332224:0@lo:400:kworker.0 00000100:00100000:1.0:1767320070.365505:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985331456:0@lo:38:kworker.0 00000100:00100000:1.0:1767320070.365563:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320070.365621:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985331584:0@lo:38:kworker.0 00000100:00100000:1.0:1767320070.365647:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320070.365671:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985331840:0@lo:38:kworker.0 00000100:00100000:1.0:1767320070.365689:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320070.365702:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985332096:0@lo:38:kworker.0 00000100:00100000:1.0:1767320070.365717:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320070.365758:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331456 00000100:00100000:1.0:1767320070.365764:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331456 00000100:00100000:1.0:1767320070.365791:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331840 00000100:00100000:1.0:1767320070.365794:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331840 00000100:00100000:1.0:1767320070.365804:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332096 00000100:00100000:1.0:1767320070.365806:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332096 00000100:00100000:1.0:1767320070.365816:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5605 00000100:00100000:1.0:1767320070.365823:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985331456:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320070.365842:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320070.365879:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985331456:12345-0@lo:38:kworker.0 Request processed in 55us (315us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320070.365886:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5605 00000100:00100000:0.0:1767320070.365892:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5606 00000100:00100000:0.0:1767320070.365897:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985331840:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320070.365901:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5607 00000100:00100000:1.0:1767320070.365904:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985332096:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320070.365907:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320070.365911:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320070.365932:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985331840:12345-0@lo:38:kworker.0 Request processed in 34us (241us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320070.365932:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985332096:12345-0@lo:38:kworker.0 Request processed in 25us (213us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320070.365938:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5607 00000100:00100000:3.0:1767320070.366020:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985331584 00000100:00100000:3.0:1767320070.366035:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985331584 00000100:00100000:3.0:1767320070.366071:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1892 00000100:00100000:3.0:1767320070.366076:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985331584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320070.366086:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320070.366114:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985331584:12345-0@lo:38:kworker.0 Request processed in 38us (467us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320070.366119:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1892 00000100:00080000:1.0:1767320070.366301:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320070.366307:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320070.366311:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985331456:0@lo:38:kworker.0 00000100:00080000:1.0:1767320070.366320:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320070.366324:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320070.366327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985331584:0@lo:38:kworker.0 00000100:00080000:1.0:1767320070.366335:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320070.366338:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320070.366341:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985331840:0@lo:38:kworker.0 00000100:00080000:1.0:1767320070.366351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320070.366354:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320070.366357:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985332096:0@lo:38:kworker.0 00000100:00100000:0.0:1767320070.380252:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5606 00010000:00010000:2.0:1767320071.003404:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320071.338565:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320071.338657:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320071.338897:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596352 00000100:00100000:0.0:1767320071.338910:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596352 00000100:00100000:0.0:1767320071.338970:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596480 00000100:00100000:0.0:1767320071.338973:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596480 00000100:00100000:0.0:1767320071.339052:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5002 00000100:00100000:0.0:1767320071.339064:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920596480:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320071.339157:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920596480:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 96us (500us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320071.339193:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5002 00000100:00100000:0.0:1767320071.339214:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5608 00000100:00100000:0.0:1767320071.339219:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920596352:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320071.339243:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320071.339270:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920596352:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 50us (719us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320071.339277:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5608 00000100:00080000:2.0:1767320071.339704:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320072.030117:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320072.926271:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985332480:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320072.926375:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320072.926543:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332480 00000100:00100000:3.0:1767320072.926555:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332480 00000100:00100000:3.0:1767320072.926651:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2452 00000100:00100000:3.0:1767320072.926662:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985332480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320072.926700:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320072.926761:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985332480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 101us (386us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320072.926770:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2452 00000100:00100000:3.0:1767320072.926839:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985332480:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320072.926994:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320072.927357:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985332608:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320072.927400:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320072.927488:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332608 00000100:00100000:1.0:1767320072.927493:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332608 00000100:00100000:1.0:1767320072.927540:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2453 00000100:00100000:1.0:1767320072.927548:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985332608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320072.927567:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320072.927616:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985332608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320072.927625:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2453 00000100:00080000:2.0:1767320072.928278:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320072.928320:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985332608:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320073.052034:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320074.076777:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320075.101054:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320075.483630:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320075.483646:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320075.483649:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320075.483654:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320075.483659:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320075.483670:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5318 00000100:00080000:2.0:1767320075.483675:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320075.483683:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320075.483750:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ed80 x1853163985332736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320075.483780:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320075.483784:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320075.483786:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320075.483789:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5318 00000100:00080000:2.0:1767320075.483792:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320075.483795:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320075.483811:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f480 x1853163985332864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320075.483856:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320075.483860:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320075.483863:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320075.483874:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5318 00000100:00080000:2.0:1767320075.483876:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320075.483880:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320075.483892:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e300 x1853163985333120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320075.483910:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320075.483913:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320075.483915:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320075.483919:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5318 00000100:00080000:2.0:1767320075.483921:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320075.483924:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320075.483954:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10a80 x1853163985333376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320075.484010:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985333632:0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.484118:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320075.484240:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320075.484247:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985333504:0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.484272:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320075.484310:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985333632 00000100:00100000:2.0:1767320075.484318:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985333632 00000100:00100000:2.0:1767320075.484394:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985333504 00000100:00100000:2.0:1767320075.484397:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985333504 00000100:00100000:2.0:1767320075.484420:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3265 00000100:00100000:2.0:1767320075.484432:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985333632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.484491:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985333632:12345-0@lo:400:kworker.0 Request processed in 62us (373us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320075.484499:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3265 00000100:00100000:2.0:1767320075.484518:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3266 00000100:00100000:2.0:1767320075.484524:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985333504:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.484551:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985333504:12345-0@lo:400:kworker.0 Request processed in 26us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320075.484557:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3266 00000100:00080000:2.0:1767320075.484582:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320075.484588:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320075.484622:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985333504:0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.484633:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985333632:0@lo:400:kworker.0 00000100:00100000:1.0:1767320075.485547:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985332736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320075.485618:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320075.485665:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985332864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320075.485690:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320075.485712:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985333120:0@lo:38:kworker.0 00000100:00100000:1.0:1767320075.485730:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320075.485743:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985333376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320075.485759:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320075.486666:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332736 00000100:00100000:2.0:1767320075.486678:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332736 00000100:00100000:3.0:1767320075.486984:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332864 00000100:00100000:3.0:1767320075.486991:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332864 00000100:00100000:3.0:1767320075.487031:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1893 00000100:00100000:3.0:1767320075.487039:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985332864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320075.487082:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320075.487228:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985332864:12345-0@lo:38:kworker.0 Request processed in 188us (1535us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320075.487236:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1893 00000100:00080000:3.0:1767320075.487600:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320075.487608:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320075.487613:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985332864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320075.487852:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985332992:0@lo:400:kworker.0 00000100:00100000:1.0:1767320075.487901:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320075.490777:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320075.490809:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985333248:0@lo:400:kworker.0 00000100:00100000:1.0:1767320075.491292:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320075.495337:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985333120 00000100:00100000:2.0:1767320075.495507:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985333120 00000100:00100000:2.0:1767320075.497112:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985333376 00000100:00100000:2.0:1767320075.497127:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985333376 00000100:00100000:2.0:1767320075.497162:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985332992 00000100:00100000:2.0:1767320075.497176:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985332992 00000100:00100000:2.0:1767320075.497232:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5003 00000100:00100000:2.0:1767320075.497247:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985332992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.498761:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985332992:12345-0@lo:400:kworker.0 Request processed in 1509us (10847us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320075.498775:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5003 00000100:00100000:2.0:1767320075.498826:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5609 00000100:00100000:2.0:1767320075.498834:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985332736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320075.498989:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320075.499244:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985332736:12345-0@lo:38:kworker.0 Request processed in 408us (13625us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320075.499252:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5609 00000100:00100000:2.0:1767320075.499268:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5610 00000100:00100000:2.0:1767320075.499274:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985333120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320075.499287:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320075.499364:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985333120:12345-0@lo:38:kworker.0 Request processed in 88us (13632us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320075.499369:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5610 00000100:00100000:2.0:1767320075.499384:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5611 00000100:00080000:3.0:1767320075.499517:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320075.499580:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320075.499587:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320075.499593:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985332736:0@lo:38:kworker.0 00000100:00080000:3.0:1767320075.499604:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320075.499608:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320075.499611:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985333120:0@lo:38:kworker.0 00000100:00100000:2.0:1767320075.499758:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985333376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320075.499773:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320075.499820:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985333376:12345-0@lo:38:kworker.0 Request processed in 63us (14059us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320075.499826:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5611 00000100:00080000:3.0:1767320075.501592:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320075.501608:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320075.501615:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985333376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320075.501835:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985332992:0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.502595:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985333248 00000100:00100000:2.0:1767320075.502605:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985333248 00000100:00100000:2.0:1767320075.502701:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5004 00000100:00100000:2.0:1767320075.502712:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985333248:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320075.502817:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985333248:12345-0@lo:400:kworker.0 Request processed in 107us (11525us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320075.502825:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5004 00000100:00080000:3.0:1767320075.502966:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320075.503026:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985333248:0@lo:400:kworker.0 00010000:00010000:1.0:1767320076.125213:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320076.462517:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320076.462627:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320076.463578:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596736 00000100:00100000:2.0:1767320076.463593:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596736 00000100:00080000:3.0:1767320076.463668:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320076.463696:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596608 00000100:00100000:2.0:1767320076.463701:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596608 00000100:00100000:2.0:1767320076.463728:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5005 00000100:00100000:2.0:1767320076.463743:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920596736:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320076.463754:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5612 00000100:00100000:3.0:1767320076.463762:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920596608:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320076.463791:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320076.463870:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920596608:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 106us (1241us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320076.463879:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5612 00000100:00100000:2.0:1767320076.465017:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920596736:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 1271us (2507us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320076.465029:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5005 00010000:00010000:2.0:1767320077.152296:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320078.044437:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320078.044470:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985333760:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320078.044563:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320078.045588:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985333888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320078.045645:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320078.045749:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985333760 00000100:00100000:0.0:1767320078.045760:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985333760 00000100:00100000:0.0:1767320078.045836:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985333888 00000100:00100000:0.0:1767320078.045840:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985333888 00000100:00100000:0.0:1767320078.045867:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2454 00000100:00100000:0.0:1767320078.045877:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985333760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320078.045949:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320078.046011:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985333760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 135us (1448us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320078.046020:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2454 00000100:00100000:0.0:1767320078.046040:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2455 00000100:00100000:0.0:1767320078.046046:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985333888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320078.046063:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320078.046321:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985333888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 274us (675us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320078.046328:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2455 00000100:00080000:2.0:1767320078.046404:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320078.046410:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320078.046451:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985333888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320078.046468:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985333760:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320078.171754:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320079.195800:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320080.221927:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320080.603342:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320080.603356:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320080.603360:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320080.603365:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320080.603370:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320080.603381:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5323 00000100:00080000:2.0:1767320080.603387:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320080.603526:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320080.603570:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12680 x1853163985334016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320080.603612:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320080.603617:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320080.603620:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320080.603624:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5323 00000100:00080000:2.0:1767320080.603627:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320080.603630:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320080.603644:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11500 x1853163985334144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320080.603756:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985334272:0@lo:400:kworker.0 00000100:00100000:1.0:1767320080.603842:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320080.603931:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320080.603935:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320080.603937:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320080.603941:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5323 00000100:00080000:2.0:1767320080.603943:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320080.603947:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320080.603971:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13480 x1853163985334400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320080.604346:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320080.604353:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985334528:0@lo:400:kworker.0 00000100:00100000:1.0:1767320080.604381:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320080.604487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320080.604491:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320080.604493:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320080.604496:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5323 00000100:00080000:2.0:1767320080.604499:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320080.604503:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320080.604517:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985334656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320080.604571:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985334912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320080.604738:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320080.604756:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320080.604762:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985334784:0@lo:400:kworker.0 00000100:00100000:2.0:1767320080.604779:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320080.604803:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334912 00000100:00100000:2.0:1767320080.604931:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334912 00000100:00100000:2.0:1767320080.605081:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334784 00000100:00100000:2.0:1767320080.605086:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334784 00000100:00080000:1.0:1767320080.605299:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320080.605304:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320080.605344:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3267 00000100:00100000:1.0:1767320080.605357:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985334912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320080.605426:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985334912:12345-0@lo:400:kworker.0 Request processed in 71us (686us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320080.605436:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3267 00000100:00100000:1.0:1767320080.605488:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985334912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320080.606432:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3268 00000100:00100000:2.0:1767320080.606445:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985334784:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320080.606509:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985334784:12345-0@lo:400:kworker.0 Request processed in 69us (1729us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320080.606516:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3268 00000100:00100000:2.0:1767320080.606572:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985334784:0@lo:400:kworker.0 00000100:00100000:0.0:1767320080.608587:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985334016:0@lo:38:kworker.0 00000100:00100000:0.0:1767320080.608703:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320080.608783:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985334144:0@lo:38:kworker.0 00000100:00100000:0.0:1767320080.608819:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320080.608828:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334272 00000100:00100000:2.0:1767320080.608854:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334272 00000100:00100000:0.0:1767320080.608866:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985334400:0@lo:38:kworker.0 00000100:00100000:0.0:1767320080.608896:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320080.608926:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985334656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320080.608930:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334528 00000100:00100000:2.0:1767320080.608935:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334528 00000100:00100000:0.0:1767320080.608960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320080.608974:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334016 00000100:00100000:2.0:1767320080.608979:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334016 00000100:00100000:2.0:1767320080.608997:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334400 00000100:00100000:2.0:1767320080.609000:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334400 00000100:00100000:0.0:1767320080.609015:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334656 00000100:00100000:2.0:1767320080.609023:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5006 00000100:00100000:0.0:1767320080.609028:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334656 00000100:00100000:2.0:1767320080.609036:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985334272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320080.609065:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5007 00000100:00100000:0.0:1767320080.609080:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985334528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320080.609207:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985334144 00000100:00100000:1.0:1767320080.609216:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985334144 00000100:00100000:1.0:1767320080.609260:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1894 00000100:00100000:3.0:1767320080.609270:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5613 00000100:00100000:1.0:1767320080.609272:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985334144:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320080.609285:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985334016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320080.609300:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767320080.609306:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320080.609401:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985334528:12345-0@lo:400:kworker.0 Request processed in 325us (5017us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320080.609410:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5007 00000100:00100000:0.0:1767320080.609435:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5614 00000100:00100000:0.0:1767320080.609439:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985334400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320080.609450:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320080.609483:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985334400:12345-0@lo:38:kworker.0 Request processed in 41us (585us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320080.609490:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5614 00000100:00100000:0.0:1767320080.609502:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5615 00000100:00100000:0.0:1767320080.609506:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985334656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320080.609515:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320080.609539:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985334656:12345-0@lo:38:kworker.0 Request processed in 32us (582us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320080.609545:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5615 00000100:00080000:0.0:1767320080.609621:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320080.609628:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320080.609633:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985334400:0@lo:38:kworker.0 00000100:00080000:0.0:1767320080.609643:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320080.609646:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320080.609649:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985334656:0@lo:38:kworker.0 00000100:00080000:0.0:1767320080.609695:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320080.609731:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985334528:0@lo:400:kworker.0 00000100:00100000:0.0:1767320080.609746:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985334272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320080.609848:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985334272:12345-0@lo:400:kworker.0 Request processed in 812us (6012us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320080.609856:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5006 00000100:00100000:1.0:1767320080.609944:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985334144:12345-0@lo:38:kworker.0 Request processed in 673us (1123us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320080.609951:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985334016:12345-0@lo:38:kworker.0 Request processed in 663us (1242us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320080.609967:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1894 00000100:00100000:3.0:1767320080.609974:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5613 00000100:00080000:0.0:1767320080.610019:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320080.610030:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320080.610035:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985334016:0@lo:38:kworker.0 00000100:00080000:0.0:1767320080.610048:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320080.610053:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320080.610057:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985334144:0@lo:38:kworker.0 00010000:00010000:0.0:1767320081.243536:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320081.581015:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320081.581115:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596992 00000100:00100000:2.0:1767320081.581130:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596992 00000100:00100000:1.0:1767320081.581133:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:0.0:1767320081.581296:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320081.581308:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920596864 00000100:00100000:3.0:1767320081.581315:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920596864 00000100:00100000:3.0:1767320081.581350:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5008 00000100:00100000:3.0:1767320081.581361:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920596992:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320081.581435:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5616 00000100:00100000:2.0:1767320081.581451:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920596864:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320081.581476:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320081.581556:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920596864:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 112us (424us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320081.581565:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5616 00000100:00100000:3.0:1767320081.581574:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920596992:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 212us (572us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320081.581583:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5008 00010000:00010000:1.0:1767320082.267605:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320083.163780:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985335040:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320083.163920:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320083.168878:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335040 00000100:00100000:0.0:1767320083.168901:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335040 00000100:00100000:0.0:1767320083.169035:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2456 00000100:00100000:0.0:1767320083.169049:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985335040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320083.169097:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320083.169215:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985335040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 170us (5296us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320083.169223:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2456 00000100:00100000:0.0:1767320083.169280:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985335168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320083.169325:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320083.169334:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320083.169355:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335168 00000100:00100000:1.0:1767320083.169359:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335168 00000100:00100000:1.0:1767320083.169444:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2457 00000100:00100000:1.0:1767320083.169453:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985335168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320083.169471:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320083.169510:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985335168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (185us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320083.169607:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985335040:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320083.169803:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985335168:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320083.169901:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320083.170118:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2457 00010000:00010000:2.0:1767320083.296116:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320084.315401:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320085.339625:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320085.726677:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320085.726692:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320085.726697:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320085.726702:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320085.726708:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320085.726720:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5328 00000100:00080000:2.0:1767320085.726727:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320085.726736:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320085.726807:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10700 x1853163985335296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320085.726841:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320085.726845:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320085.726847:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320085.726850:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5328 00000100:00080000:2.0:1767320085.726853:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320085.726857:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320085.726875:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11500 x1853163985335424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320085.726924:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320085.726928:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320085.726930:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320085.726934:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5328 00000100:00080000:2.0:1767320085.726936:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320085.726940:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320085.726951:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10000 x1853163985335680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320085.726967:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320085.726971:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320085.726973:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320085.726977:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5328 00000100:00080000:2.0:1767320085.726979:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320085.726982:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320085.726992:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985335936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320085.727055:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985336064:0@lo:400:kworker.0 00000100:00100000:3.0:1767320085.727059:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985335552:0@lo:400:kworker.0 00000100:00100000:1.0:1767320085.727146:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985335808:0@lo:400:kworker.0 00000100:00100000:0.0:1767320085.727220:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985336192:0@lo:400:kworker.0 00000100:00100000:3.0:1767320085.733554:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320085.733757:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335552 00000100:00100000:3.0:1767320085.733770:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335552 00000100:00100000:1.0:1767320085.733799:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320085.733852:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335808 00000100:00100000:3.0:1767320085.733856:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335808 00000100:00100000:1.0:1767320085.733880:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5009 00000100:00100000:3.0:1767320085.733891:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5010 00000100:00100000:1.0:1767320085.733898:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985335552:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320085.733904:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985335808:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320085.733981:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985335552:12345-0@lo:400:kworker.0 Request processed in 88us (439us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320085.733988:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985335808:12345-0@lo:400:kworker.0 Request processed in 87us (188us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320085.733995:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5009 00000100:00100000:3.0:1767320085.734002:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5010 00000100:00080000:3.0:1767320085.734029:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320085.734058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985335296:0@lo:38:kworker.0 00000100:00100000:1.0:1767320085.734070:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985335808:0@lo:400:kworker.0 00000100:00100000:3.0:1767320085.734097:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320085.734112:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985335552:0@lo:400:kworker.0 00000100:00100000:3.0:1767320085.734136:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985335424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320085.734141:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335296 00000100:00100000:1.0:1767320085.734146:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335296 00000100:00100000:1.0:1767320085.734193:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5617 00000100:00100000:1.0:1767320085.734198:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985335296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320085.734223:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320085.734260:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985335296:12345-0@lo:38:kworker.0 Request processed in 60us (163us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320085.734266:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5617 00000100:00100000:3.0:1767320085.737828:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320085.737925:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985335680:0@lo:38:kworker.0 00000100:00100000:3.0:1767320085.737964:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320085.737984:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985335936:0@lo:38:kworker.0 00000100:00100000:3.0:1767320085.738006:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320085.738062:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320085.738069:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320085.738074:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985335296:0@lo:38:kworker.0 00000100:00100000:3.0:1767320085.738158:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335680 00000100:00100000:3.0:1767320085.738182:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335680 00000100:00100000:3.0:1767320085.738226:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335936 00000100:00100000:3.0:1767320085.738229:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335936 00000100:00100000:3.0:1767320085.738254:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5618 00000100:00100000:3.0:1767320085.738263:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985335680:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320085.738292:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320085.738345:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985335680:12345-0@lo:38:kworker.0 Request processed in 81us (379us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320085.738353:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5618 00000100:00100000:3.0:1767320085.738371:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5619 00000100:00100000:3.0:1767320085.738376:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985335936:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320085.738385:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320085.738409:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985335936:12345-0@lo:38:kworker.0 Request processed in 31us (402us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320085.738414:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5619 00000100:00080000:3.0:1767320085.738446:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320085.738452:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320085.738454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985335680:0@lo:38:kworker.0 00000100:00080000:3.0:1767320085.738462:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320085.738466:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320085.738469:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985335936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320085.739265:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985335424 00000100:00100000:1.0:1767320085.739275:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985335424 00000100:00100000:1.0:1767320085.739321:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1895 00000100:00100000:1.0:1767320085.739329:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985335424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320085.739346:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320085.739418:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985335424:12345-0@lo:38:kworker.0 Request processed in 88us (1596us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320085.739425:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1895 00000100:00080000:3.0:1767320085.740670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320085.740681:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320085.740688:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985335424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320085.744297:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320085.744558:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985336064 00000100:00100000:3.0:1767320085.744571:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985336064 00000100:00100000:3.0:1767320085.744670:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3269 00000100:00100000:3.0:1767320085.744682:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985336064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320085.744772:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985336064:12345-0@lo:400:kworker.0 Request processed in 92us (488us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320085.744781:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3269 00000100:00100000:2.0:1767320085.745697:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985336064:0@lo:400:kworker.0 00000100:00080000:1.0:1767320085.745832:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320085.750971:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320085.758687:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985336192 00000100:00100000:3.0:1767320085.758711:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985336192 00000100:00100000:3.0:1767320085.758835:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3270 00000100:00100000:3.0:1767320085.758847:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985336192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320085.758985:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985336192:12345-0@lo:400:kworker.0 Request processed in 140us (8027us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320085.758993:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3270 00000100:00080000:1.0:1767320085.759106:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320085.759480:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985336192:0@lo:400:kworker.0 00010000:00010000:0.0:1767320086.363511:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320086.701219:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320086.703381:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320086.711564:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920597120 00000100:00100000:1.0:1767320086.711590:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920597120 00000100:00100000:1.0:1767320086.711687:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920597248 00000100:00100000:1.0:1767320086.711691:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920597248 00000100:00100000:1.0:1767320086.711854:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5011 00000100:00100000:1.0:1767320086.711867:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920597248:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320086.712017:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920597248:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 152us (8646us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320086.712026:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5011 00000100:00100000:1.0:1767320086.712049:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5620 00000100:00100000:1.0:1767320086.712055:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920597120:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320086.712080:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320086.712108:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920597120:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (10947us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320086.712115:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5620 00000100:00080000:0.0:1767320086.721299:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320087.387418:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320088.284927:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320088.284960:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985336320:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320088.285081:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320088.285340:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985336320 00000100:00100000:0.0:1767320088.285355:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985336320 00000100:00100000:0.0:1767320088.285458:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2458 00000100:00100000:0.0:1767320088.285470:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985336320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320088.285712:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320088.285778:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985336320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 310us (696us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320088.285787:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2458 00000100:00100000:3.0:1767320088.286332:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320088.286350:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985336448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320088.286406:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320088.286509:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320088.286551:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985336320:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320088.287015:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985336448 00000100:00100000:0.0:1767320088.287024:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985336448 00000100:00100000:0.0:1767320088.287098:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2459 00000100:00100000:0.0:1767320088.287108:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985336448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320088.287139:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767320088.287184:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320088.287284:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985336448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 177us (876us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320088.287293:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2459 00000100:00100000:3.0:1767320088.288451:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985336448:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320088.411566:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320089.439736:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320090.459541:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320090.797925:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320090.798186:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920597632 00000100:00100000:2.0:1767320090.798200:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920597632 00000100:00100000:2.0:1767320090.798367:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5012 00000100:00100000:2.0:1767320090.798390:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920597632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320090.798504:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920597632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 118us (595us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320090.798513:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5012 00000100:00080000:0.0:1767320090.798581:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767320090.798717:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920597376 offset 0 length 520: 4 00000100:00100000:0.0:1767320090.798838:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320090.804958:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920597504 00000100:00100000:1.0:1767320090.804983:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920597504 00000100:00100000:1.0:1767320090.805559:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5621 00000100:00100000:1.0:1767320090.805570:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920597504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320090.805603:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320090.805722:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920597504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 150us (6881us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.805730:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5621 00000100:00100000:3.0:1767320090.843864:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320065/real 1767320065] req@ffff8ce1b357aa00 x1853163985330048/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320090 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320090.843894:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b357aa00 x1853163985330048/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320090 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320090.843910:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320090.847597:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320090.847609:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985330048:0@lo:250:kworker.0 00000100:00080000:0.0:1767320090.848715:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.848727:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320090.848732:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.849432:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5313 00000100:00080000:0.0:1767320090.849440:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.849450:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320090.849503:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985336576/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320090.849537:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.849542:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.849544:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.849548:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5333 00000100:00080000:0.0:1767320090.849551:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.849556:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320090.849575:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404e00 x1853163985336704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320090.849585:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.849589:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.849591:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.849595:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5333 00000100:00080000:0.0:1767320090.849598:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.849606:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320090.849606:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985336576:0@lo:250:kworker.0 00000100:00100000:0.0:1767320090.849633:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3480 x1853163985336832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:3.0:1767320090.849687:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985336576 offset 0 length 520: 4 00000100:00100000:3.0:1767320090.849733:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985336704:0@lo:38:kworker.0 00000100:00100000:2.0:1767320090.849759:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985336960:0@lo:400:kworker.0 00000100:00100000:3.0:1767320090.849773:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320090.849813:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985336832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320090.849832:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320090.849862:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985336832 00000100:00100000:3.0:1767320090.849868:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985336832 00000100:00100000:3.0:1767320090.849903:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1896 00000100:00100000:3.0:1767320090.849915:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985336832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320090.849935:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320090.849972:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320090.850013:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985336832:12345-0@lo:38:kworker.0 Request processed in 100us (177us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320090.850020:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1896 00000100:00080000:0.0:1767320090.850049:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.850055:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.850058:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.850062:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5333 00000100:00080000:0.0:1767320090.850065:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.850070:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320090.850088:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0700 x1853163985337088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320090.850211:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320090.850220:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985337216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320090.850260:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320090.851568:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320090.851580:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320090.851585:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985336832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320090.851616:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985337088:0@lo:38:kworker.0 00000100:00100000:3.0:1767320090.851679:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320090.852435:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.852445:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.852448:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.852456:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5333 00000100:00080000:0.0:1767320090.852460:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.852466:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320090.852506:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0000 x1853163985337344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320090.852584:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985337472:0@lo:400:kworker.0 00000100:00100000:0.0:1767320090.852641:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320090.852702:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337472 00000100:00100000:0.0:1767320090.852712:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337472 00000100:00100000:1.0:1767320090.853730:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985336704 00000100:00100000:1.0:1767320090.853746:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985336704 00000100:00100000:1.0:1767320090.853805:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985336960 00000100:00100000:1.0:1767320090.853808:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985336960 00000100:00100000:0.0:1767320090.853860:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3271 00000100:00100000:1.0:1767320090.853864:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337216 00000100:00100000:1.0:1767320090.853871:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337216 00000100:00100000:0.0:1767320090.853880:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985337472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.853914:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337088 00000100:00100000:1.0:1767320090.853921:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337088 00000100:00100000:1.0:1767320090.853939:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5013 00000100:00100000:1.0:1767320090.853951:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985336960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320090.853993:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985337472:12345-0@lo:400:kworker.0 Request processed in 120us (1348us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320090.854004:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3271 00000100:00100000:1.0:1767320090.854046:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985336960:12345-0@lo:400:kworker.0 Request processed in 96us (4075us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320090.854072:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5013 00000100:00100000:1.0:1767320090.854090:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5014 00000100:00100000:1.0:1767320090.854096:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985337216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.854126:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985337216:12345-0@lo:400:kworker.0 Request processed in 30us (3867us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320090.854132:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5014 00000100:00100000:1.0:1767320090.854145:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5622 00000100:00100000:1.0:1767320090.854149:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985336704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320090.854189:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320090.854211:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985336704:12345-0@lo:38:kworker.0 Request processed in 62us (4449us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.854216:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5622 00000100:00100000:1.0:1767320090.854224:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5623 00000100:00100000:1.0:1767320090.854226:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985337088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320090.854232:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320090.854245:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985337088:12345-0@lo:38:kworker.0 Request processed in 18us (2568us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.854248:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5623 00000100:00080000:1.0:1767320090.854285:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320090.854288:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320090.854619:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985337472:0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.854636:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320090.854658:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985337600:0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.854693:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320090.854718:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337600 00000100:00100000:1.0:1767320090.854720:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337600 00000100:00100000:1.0:1767320090.854743:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3272 00000100:00100000:1.0:1767320090.854747:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985337600:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.854769:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985337600:12345-0@lo:400:kworker.0 Request processed in 22us (77us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320090.854773:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3272 00000100:00080000:1.0:1767320090.854782:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320090.854791:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985337600:0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.854821:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985337216:0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.854827:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985336960:0@lo:400:kworker.0 00000100:00100000:0.0:1767320090.855709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985337344:0@lo:38:kworker.0 00000100:00100000:0.0:1767320090.856028:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320090.856081:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320090.856087:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320090.856091:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985337088:0@lo:38:kworker.0 00000100:00080000:0.0:1767320090.856106:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320090.856111:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320090.856114:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985336704:0@lo:38:kworker.0 00000100:00100000:1.0:1767320090.857452:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337344 00000100:00100000:1.0:1767320090.857466:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337344 00000100:00100000:1.0:1767320090.857527:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5624 00000100:00100000:1.0:1767320090.857534:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985337344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320090.857555:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320090.857624:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985337344:12345-0@lo:38:kworker.0 Request processed in 90us (1608us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.857632:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5624 00000100:00080000:0.0:1767320090.857709:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320090.857719:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320090.857724:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985337344:0@lo:38:kworker.0 00010000:00010000:2.0:1767320091.483527:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320092.507907:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320093.403813:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320093.403838:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985337728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320093.403907:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320093.404077:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337728 00000100:00100000:0.0:1767320093.404086:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337728 00000100:00100000:0.0:1767320093.404189:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2460 00000100:00100000:0.0:1767320093.404202:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985337728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320093.404231:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320093.404286:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985337728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (377us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320093.404294:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2460 00000100:00100000:3.0:1767320093.404345:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985337728:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320093.404367:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320093.408086:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985337856:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320093.408198:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320093.408273:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337856 00000100:00100000:0.0:1767320093.408287:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337856 00000100:00100000:0.0:1767320093.408387:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2461 00000100:00100000:0.0:1767320093.408398:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985337856:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320093.408548:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:0.0:1767320093.408624:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320093.408712:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985337856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 315us (549us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320093.408721:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2461 00000100:00100000:2.0:1767320093.408875:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985337856:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320093.532371:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320094.555527:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320095.579450:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320095.913470:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320095.913665:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320095.913893:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920597888 00000100:00100000:0.0:1767320095.913908:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920597888 00000100:00100000:0.0:1767320095.914016:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920597760 00000100:00100000:0.0:1767320095.914021:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920597760 00000100:00100000:0.0:1767320095.914061:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5015 00000100:00100000:0.0:1767320095.914075:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920597888:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320095.914243:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920597888:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 173us (787us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320095.914252:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5015 00000100:00100000:0.0:1767320095.914275:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5625 00000100:00100000:0.0:1767320095.914279:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920597760:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320095.914307:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320095.914337:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920597760:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (673us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320095.914343:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5625 00000100:00080000:1.0:1767320095.914627:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320095.963407:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320095.963422:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320095.963426:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320095.963431:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320095.963437:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320095.963448:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5339 00000100:00080000:2.0:1767320095.963455:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320095.963463:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320095.963524:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985337984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320095.963562:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320095.963567:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320095.963569:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320095.963573:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5339 00000100:00080000:2.0:1767320095.963576:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320095.963580:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320095.963594:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10000 x1853163985338112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320095.963666:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985338240:0@lo:400:kworker.0 00000100:00100000:1.0:1767320095.963741:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320095.963855:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985338240 00000100:00100000:3.0:1767320095.963864:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985338240 00000100:00100000:3.0:1767320095.963935:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5016 00000100:00100000:3.0:1767320095.963945:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985338240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320095.964045:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985337984:0@lo:38:kworker.0 00000100:00100000:0.0:1767320095.964070:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320095.964085:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985338112:0@lo:38:kworker.0 00000100:00100000:0.0:1767320095.964102:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320095.964129:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985337984 00000100:00100000:0.0:1767320095.964132:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985337984 00000100:00100000:0.0:1767320095.964151:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5626 00000100:00100000:0.0:1767320095.964154:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985337984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320095.964195:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320095.964221:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985337984:12345-0@lo:38:kworker.0 Request processed in 66us (149us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320095.964226:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5626 00000100:00080000:0.0:1767320095.964257:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320095.964261:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320095.964264:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985337984:0@lo:38:kworker.0 00000100:00080000:0.0:1767320095.964297:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320095.964310:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985338112 00000100:00100000:0.0:1767320095.964312:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985338112 00000100:00100000:0.0:1767320095.964327:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1897 00000100:00100000:0.0:1767320095.964330:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985338112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320095.964337:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320095.964355:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985338112:12345-0@lo:38:kworker.0 Request processed in 23us (252us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320095.964359:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1897 00000100:00080000:0.0:1767320095.964372:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320095.964374:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320095.964376:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985338112:0@lo:38:kworker.0 00000100:00080000:2.0:1767320095.964649:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320095.964653:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320095.964655:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320095.964659:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5339 00000100:00080000:2.0:1767320095.964661:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320095.964664:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320095.964681:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12680 x1853163985338368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320095.964733:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985338496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320095.964756:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320095.964779:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985338496 00000100:00100000:0.0:1767320095.964781:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985338496 00000100:00100000:0.0:1767320095.964803:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5017 00000100:00100000:0.0:1767320095.964807:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985338496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320095.964830:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985338496:12345-0@lo:400:kworker.0 Request processed in 23us (74us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320095.964834:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5017 00000100:00080000:0.0:1767320095.964843:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320095.964856:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985338496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320095.964876:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985338368:0@lo:38:kworker.0 00000100:00100000:0.0:1767320095.964892:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320095.964912:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985338368 00000100:00100000:0.0:1767320095.964916:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985338368 00000100:00100000:0.0:1767320095.964933:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5627 00000100:00100000:0.0:1767320095.964937:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985338368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320095.964949:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320095.964977:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985338368:12345-0@lo:38:kworker.0 Request processed in 38us (83us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320095.964983:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5627 00000100:00080000:0.0:1767320095.965007:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320095.965010:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320095.965013:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985338368:0@lo:38:kworker.0 00000100:00100000:3.0:1767320095.966832:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985338240:12345-0@lo:400:kworker.0 Request processed in 2877us (3077us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320095.966848:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5016 00000100:00080000:2.0:1767320095.967304:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320095.967311:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320095.967315:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320095.967320:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5339 00000100:00080000:2.0:1767320095.967325:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320095.967331:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320095.967378:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12680 x1853163985338624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320095.967473:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985338752:0@lo:400:kworker.0 00000100:00100000:2.0:1767320095.967546:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320095.967593:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320095.967600:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985338880:0@lo:400:kworker.0 00000100:00100000:2.0:1767320095.967627:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320095.967663:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985338752 00000100:00100000:2.0:1767320095.967672:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985338752 00000100:00100000:2.0:1767320095.967730:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985338880 00000100:00100000:2.0:1767320095.967734:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985338880 00000100:00100000:2.0:1767320095.967759:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3273 00000100:00100000:2.0:1767320095.967769:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985338752:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320095.967821:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985338752:12345-0@lo:400:kworker.0 Request processed in 54us (274us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320095.967829:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3273 00000100:00100000:2.0:1767320095.967844:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3274 00000100:00100000:2.0:1767320095.967850:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985338880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320095.968074:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985338624:0@lo:38:kworker.0 00000100:00080000:0.0:1767320095.968477:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320095.968484:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320095.974624:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985338240:0@lo:400:kworker.0 00000100:00100000:1.0:1767320095.974755:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985338752:0@lo:400:kworker.0 00000100:00100000:3.0:1767320095.976248:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320095.976450:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985338624 00000100:00100000:0.0:1767320095.976461:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985338624 00000100:00100000:0.0:1767320095.976522:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5628 00000100:00100000:0.0:1767320095.976531:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985338624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320095.976568:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320095.976642:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985338624:12345-0@lo:38:kworker.0 Request processed in 110us (402us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320095.976650:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5628 00000100:00100000:2.0:1767320095.976830:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985338880:12345-0@lo:400:kworker.0 Request processed in 8978us (9200us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320095.976841:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3274 00000100:00100000:0.0:1767320095.977934:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985338880:0@lo:400:kworker.0 00000100:00080000:3.0:1767320095.978225:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320095.978232:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320095.978236:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985338624:0@lo:38:kworker.0 00010000:00010000:1.0:1767320096.603574:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320097.627358:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320098.524328:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320098.524364:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985339008:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320098.524451:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320098.524696:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985339136:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320098.524721:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339008 00000100:00100000:1.0:1767320098.524745:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339008 00000100:00100000:3.0:1767320098.524768:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320098.524836:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339136 00000100:00100000:1.0:1767320098.524842:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339136 00000100:00100000:1.0:1767320098.524870:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2462 00000100:00100000:1.0:1767320098.524880:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985339008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320098.524912:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320098.524996:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985339008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 115us (541us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320098.525004:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2462 00000100:00100000:3.0:1767320098.525013:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2463 00000100:00100000:3.0:1767320098.525033:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985339136:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320098.525066:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320098.525112:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985339136:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (345us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320098.525120:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2463 00000100:00100000:3.0:1767320098.525566:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985339136:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320098.525592:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985339008:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320098.525820:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320098.525827:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767320098.651450:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320099.675569:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320100.699385:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320101.035143:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320101.035260:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320101.042642:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920598144 00000100:00100000:1.0:1767320101.042659:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920598144 00000100:00100000:1.0:1767320101.043257:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920598016 00000100:00100000:1.0:1767320101.043263:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920598016 00000100:00100000:1.0:1767320101.043290:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5018 00000100:00100000:1.0:1767320101.043303:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920598144:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320101.043388:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920598144:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (8255us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320101.043396:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5018 00000100:00100000:1.0:1767320101.043416:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5629 00000100:00100000:1.0:1767320101.043421:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920598016:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320101.043441:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320101.043481:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920598016:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 57us (8219us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320101.043490:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5629 00000100:00080000:0.0:1767320101.044567:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320101.084632:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320101.084643:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320101.084646:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320101.084650:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320101.084654:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320101.084662:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5344 00000100:00080000:0.0:1767320101.084667:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320101.084673:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320101.084711:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2d80 x1853163985339264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320101.084740:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320101.084743:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320101.084745:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320101.084748:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5344 00000100:00080000:0.0:1767320101.084750:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320101.084932:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985339264:0@lo:38:kworker.0 00000100:00100000:3.0:1767320101.085007:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320101.085269:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339264 00000100:00100000:1.0:1767320101.085279:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339264 00000100:00100000:1.0:1767320101.085324:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5630 00000100:00100000:1.0:1767320101.085331:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985339264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320101.085352:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320101.085403:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985339264:12345-0@lo:38:kworker.0 Request processed in 71us (395us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320101.085410:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5630 00000100:00080000:0.0:1767320101.085526:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320101.085546:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3100 x1853163985339392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320101.085609:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:0.0:1767320101.085898:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320101.085902:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320101.085904:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320101.085910:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5344 00000100:00080000:0.0:1767320101.085913:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320101.085917:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320101.085933:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1180 x1853163985339648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320101.086001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985339392:0@lo:38:kworker.0 00000100:00100000:3.0:1767320101.086040:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320101.086081:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985339648:0@lo:38:kworker.0 00000100:00100000:3.0:1767320101.086101:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320101.086133:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320101.086138:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320101.086141:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985339264:0@lo:38:kworker.0 00000100:00100000:3.0:1767320101.086203:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339648 00000100:00100000:3.0:1767320101.086208:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339648 00000100:00100000:3.0:1767320101.086232:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5631 00000100:00100000:3.0:1767320101.086238:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985339648:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320101.086252:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320101.086285:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985339648:12345-0@lo:38:kworker.0 Request processed in 46us (183us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320101.086292:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5631 00000100:00080000:3.0:1767320101.086315:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320101.086318:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320101.086322:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985339648:0@lo:38:kworker.0 00000100:00100000:1.0:1767320101.088205:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339392 00000100:00100000:1.0:1767320101.088220:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339392 00000100:00100000:1.0:1767320101.088279:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1898 00000100:00100000:1.0:1767320101.088286:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985339392:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320101.088307:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320101.088378:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985339392:12345-0@lo:38:kworker.0 Request processed in 91us (2336us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320101.088384:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1898 00000100:00080000:3.0:1767320101.088522:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320101.088528:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320101.088531:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985339392:0@lo:38:kworker.0 00000100:00100000:2.0:1767320101.088645:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985339776:0@lo:400:kworker.0 00000100:00100000:2.0:1767320101.088684:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320101.088703:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985339520:0@lo:400:kworker.0 00000100:00100000:2.0:1767320101.088721:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320101.088925:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339776 00000100:00100000:3.0:1767320101.088930:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339776 00000100:00100000:3.0:1767320101.088986:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339520 00000100:00100000:3.0:1767320101.088990:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339520 00000100:00100000:3.0:1767320101.089011:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5019 00000100:00100000:3.0:1767320101.089018:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985339776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320101.089063:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985339776:12345-0@lo:400:kworker.0 Request processed in 46us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320101.089069:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5019 00000100:00100000:3.0:1767320101.089083:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5020 00000100:00100000:3.0:1767320101.089089:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985339520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320101.089116:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985339520:12345-0@lo:400:kworker.0 Request processed in 27us (394us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320101.089122:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5020 00000100:00080000:3.0:1767320101.089150:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320101.089375:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985339776:0@lo:400:kworker.0 00000100:00100000:2.0:1767320101.089386:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985339520:0@lo:400:kworker.0 00000100:00080000:0.0:1767320101.089850:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320101.089856:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320101.089859:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320101.089863:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5344 00000100:00080000:0.0:1767320101.089867:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320101.089871:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320101.089903:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2680 x1853163985339904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320101.089982:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320101.090026:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985339904:0@lo:38:kworker.0 00000100:00100000:3.0:1767320101.090064:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320101.090436:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985339904 00000100:00100000:2.0:1767320101.090442:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985339904 00000100:00100000:2.0:1767320101.090493:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5632 00000100:00100000:2.0:1767320101.090499:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985339904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320101.090525:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320101.090568:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985339904:12345-0@lo:38:kworker.0 Request processed in 67us (502us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320101.090574:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5632 00000100:00080000:2.0:1767320101.090607:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320101.090610:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320101.090614:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985339904:0@lo:38:kworker.0 00000100:00100000:1.0:1767320101.095920:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985340160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320101.096249:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320101.096286:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985340032:0@lo:400:kworker.0 00000100:00100000:1.0:1767320101.096309:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320101.096433:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340160 00000100:00100000:3.0:1767320101.096441:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340160 00000100:00100000:3.0:1767320101.096521:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340032 00000100:00100000:3.0:1767320101.096525:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340032 00000100:00100000:3.0:1767320101.096547:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3275 00000100:00100000:3.0:1767320101.096556:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985340160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320101.096611:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985340160:12345-0@lo:400:kworker.0 Request processed in 56us (362us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320101.096619:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3275 00000100:00100000:3.0:1767320101.096645:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3276 00000100:00100000:3.0:1767320101.096651:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985340032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320101.096677:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985340032:12345-0@lo:400:kworker.0 Request processed in 26us (367us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320101.096683:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3276 00000100:00080000:1.0:1767320101.097510:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320101.097525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320101.097567:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985340160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320101.097583:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985340032:0@lo:400:kworker.0 00010000:00010000:2.0:1767320101.723365:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320102.747495:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320103.646252:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985340288:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320103.646325:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320103.646445:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340288 00000100:00100000:1.0:1767320103.646454:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340288 00000100:00080000:3.0:1767320103.646513:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320103.646524:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2464 00000100:00100000:1.0:1767320103.646535:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985340288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320103.646566:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320103.647299:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985340416:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320103.648006:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985340288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1471us (1678us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320103.648018:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2464 00000100:00100000:2.0:1767320103.648248:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985340288:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320103.649515:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320103.649581:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340416 00000100:00100000:1.0:1767320103.649589:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340416 00000100:00100000:1.0:1767320103.649656:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2465 00000100:00100000:1.0:1767320103.649665:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985340416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320103.649695:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320103.649754:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985340416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320103.649760:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2465 00000100:00080000:3.0:1767320103.649823:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320103.649901:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985340416:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320103.771718:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320104.798210:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320105.819518:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320106.153626:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320106.154047:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920598400 00000100:00100000:0.0:1767320106.154057:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920598400 00000100:00080000:3.0:1767320106.154186:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320106.154265:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5021 00000100:00100000:0.0:1767320106.154278:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920598400:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320106.154371:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920598400:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (746us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320106.154379:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5021 00000100:00100000:3.0:1767320106.155918:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320106.170464:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920598272 00000100:00100000:2.0:1767320106.170482:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920598272 00000100:00100000:0.0:1767320106.173752:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5633 00000100:00100000:0.0:1767320106.173762:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920598272:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320106.173786:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320106.173869:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920598272:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 106us (17958us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320106.173877:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5633 00000100:00080000:1.0:1767320106.204128:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320106.204138:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320106.204142:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320106.204145:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320106.204149:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320106.204157:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5349 00000100:00080000:1.0:1767320106.204161:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320106.204190:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320106.204230:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13800 x1853163985340544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320106.204259:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320106.204262:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320106.204264:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320106.204267:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5349 00000100:00080000:1.0:1767320106.204269:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320106.204272:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320106.204283:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10700 x1853163985340672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320106.204326:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320106.204330:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320106.204332:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320106.204336:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5349 00000100:00080000:1.0:1767320106.204339:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320106.204343:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320106.204350:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985340800:0@lo:400:kworker.0 00000100:00100000:1.0:1767320106.204356:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985340928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320106.204385:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320106.204389:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320106.204391:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320106.204394:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5349 00000100:00080000:1.0:1767320106.204396:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320106.204400:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320106.204431:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320106.204497:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320106.204504:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985341056:0@lo:400:kworker.0 00000100:00100000:2.0:1767320106.204526:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320106.204568:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340800 00000100:00100000:2.0:1767320106.204576:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340800 00000100:00100000:2.0:1767320106.204623:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341056 00000100:00100000:2.0:1767320106.204627:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341056 00000100:00100000:2.0:1767320106.204648:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5022 00000100:00100000:2.0:1767320106.204670:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985340800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320106.204686:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4380 x1853163985341184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320106.204726:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985340800:12345-0@lo:400:kworker.0 Request processed in 70us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320106.204736:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5022 00000100:00100000:1.0:1767320106.204747:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985341312:0@lo:400:kworker.0 00000100:00100000:2.0:1767320106.204757:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5023 00000100:00100000:2.0:1767320106.204765:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985341056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320106.204786:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320106.204814:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320106.204820:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985341440:0@lo:400:kworker.0 00000100:00100000:1.0:1767320106.204842:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320106.204872:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341312 00000100:00100000:1.0:1767320106.204876:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341312 00000100:00100000:1.0:1767320106.204908:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341440 00000100:00100000:1.0:1767320106.204911:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341440 00000100:00100000:3.0:1767320106.204930:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3277 00000100:00100000:1.0:1767320106.204939:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3278 00000100:00100000:3.0:1767320106.204944:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985341312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320106.204951:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985341440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320106.204986:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985341312:12345-0@lo:400:kworker.0 Request processed in 47us (201us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320106.204994:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3277 00000100:00100000:1.0:1767320106.205007:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985341440:12345-0@lo:400:kworker.0 Request processed in 60us (163us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320106.205015:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3278 00000100:00080000:3.0:1767320106.205023:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320106.205029:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320106.205032:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320106.205063:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985341440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320106.205078:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985341312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320106.205124:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985341056:0@lo:400:kworker.0 00000100:00100000:0.0:1767320106.205130:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985340544:0@lo:38:kworker.0 00000100:00100000:3.0:1767320106.205150:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985340800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320106.205189:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320106.205216:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985340672:0@lo:38:kworker.0 00000100:00100000:0.0:1767320106.205240:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320106.205271:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985340928:0@lo:38:kworker.0 00000100:00100000:0.0:1767320106.205290:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320106.205303:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985341184:0@lo:38:kworker.0 00000100:00100000:0.0:1767320106.205319:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320106.205347:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340544 00000100:00100000:0.0:1767320106.205351:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340544 00000100:00100000:0.0:1767320106.205377:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340928 00000100:00100000:0.0:1767320106.205380:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340928 00000100:00100000:0.0:1767320106.205388:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341184 00000100:00100000:0.0:1767320106.205390:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341184 00000100:00100000:0.0:1767320106.205398:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5634 00000100:00100000:0.0:1767320106.205403:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985340544:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320106.205419:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320106.205451:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985340544:12345-0@lo:38:kworker.0 Request processed in 47us (288us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320106.205458:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5634 00000100:00080000:0.0:1767320106.205479:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320106.205484:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320106.205487:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985340544:0@lo:38:kworker.0 00000100:00100000:0.0:1767320106.205504:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5635 00000100:00100000:0.0:1767320106.205508:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985340928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320106.205516:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320106.205541:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985340928:12345-0@lo:38:kworker.0 Request processed in 32us (250us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320106.205547:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5635 00000100:00080000:0.0:1767320106.205565:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320106.205569:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320106.205571:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985340928:0@lo:38:kworker.0 00000100:00100000:0.0:1767320106.205589:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5636 00000100:00100000:0.0:1767320106.205593:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985341184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320106.205602:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320106.205626:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985341184:12345-0@lo:38:kworker.0 Request processed in 32us (306us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320106.205632:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5636 00000100:00080000:0.0:1767320106.205650:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320106.205653:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320106.205656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985341184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320106.205804:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985340672 00000100:00100000:3.0:1767320106.205809:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985340672 00000100:00100000:3.0:1767320106.205837:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1899 00000100:00100000:3.0:1767320106.205841:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985340672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320106.205850:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320106.205881:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985340672:12345-0@lo:38:kworker.0 Request processed in 38us (640us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320106.205887:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1899 00000100:00080000:3.0:1767320106.205913:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320106.205917:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320106.205920:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985340672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320106.207078:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985341056:12345-0@lo:400:kworker.0 Request processed in 35us (272us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320106.207093:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5023 00010000:00010000:1.0:1767320106.843400:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320107.870060:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320108.763413:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320108.763449:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985341568:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320108.763757:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320108.763839:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341568 00000100:00100000:1.0:1767320108.763846:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341568 00000100:00100000:1.0:1767320108.763897:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2466 00000100:00100000:1.0:1767320108.763904:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985341568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320108.763928:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320108.763965:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985341568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (208us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320108.763970:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2466 00000100:00100000:2.0:1767320108.764108:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320108.764116:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985341696:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320108.764137:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320108.764218:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341696 00000100:00100000:1.0:1767320108.764223:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341696 00000100:00100000:1.0:1767320108.764253:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2467 00000100:00100000:1.0:1767320108.764259:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985341696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320108.764273:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320108.764320:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985341696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (172us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320108.764328:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2467 00000100:00100000:0.0:1767320108.764438:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985341696:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320108.764557:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320108.764562:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320108.764585:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985341568:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320108.892040:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320109.915394:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320110.944953:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320111.274837:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320111.274998:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920598656 00000100:00100000:0.0:1767320111.275008:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920598656 00000100:00100000:0.0:1767320111.275083:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5024 00000100:00100000:0.0:1767320111.275094:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920598656:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320111.275219:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920598656:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 126us (391us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320111.275227:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5024 00000100:00080000:0.0:1767320111.278611:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320111.279436:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320111.280105:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920598528 00000100:00100000:2.0:1767320111.280117:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920598528 00000100:00100000:2.0:1767320111.280233:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5637 00000100:00100000:2.0:1767320111.280242:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920598528:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320111.280278:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320111.280353:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920598528:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 110us (922us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320111.280361:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5637 00000100:00080000:1.0:1767320111.323754:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320111.323767:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320111.323772:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320111.323787:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320111.323793:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320111.323805:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5354 00000100:00080000:1.0:1767320111.323811:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320111.323822:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320111.323872:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6d80 x1853163985341824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320111.326420:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320111.326431:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320111.326435:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320111.326443:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5354 00000100:00080000:1.0:1767320111.326449:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320111.326457:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320111.326497:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7100 x1853163985341952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320111.326525:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985341824:0@lo:38:kworker.0 00000100:00100000:0.0:1767320111.326627:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320111.326684:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320111.326691:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320111.326693:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320111.326700:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5354 00000100:00100000:0.0:1767320111.326706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985341952:0@lo:38:kworker.0 00000100:00080000:1.0:1767320111.326706:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320111.326718:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320111.326736:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985342208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320111.326751:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320111.326763:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320111.326767:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320111.326770:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320111.326775:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5354 00000100:00080000:1.0:1767320111.326779:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320111.326792:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985342208:0@lo:38:kworker.0 00000100:00100000:0.0:1767320111.326818:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320111.326857:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320111.326863:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341824 00000100:00100000:0.0:1767320111.326875:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341824 00000100:00100000:1.0:1767320111.326876:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985342464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320111.326925:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342208 00000100:00100000:0.0:1767320111.326931:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342208 00000100:00100000:0.0:1767320111.326968:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5638 00000100:00100000:0.0:1767320111.326978:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985341824:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320111.326986:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985342592:0@lo:400:kworker.0 00010000:00020000:0.0:1767320111.327006:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320111.327060:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320111.327067:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985341824:12345-0@lo:38:kworker.0 Request processed in 86us (436us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320111.327085:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5638 00000100:00100000:0.0:1767320111.327107:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5639 00000100:00100000:0.0:1767320111.327112:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985342208:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320111.327121:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320111.327150:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985342208:12345-0@lo:38:kworker.0 Request processed in 36us (331us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320111.327159:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5639 00000100:00100000:3.0:1767320111.327555:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985342720:0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.327594:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320111.327634:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342592 00000100:00100000:3.0:1767320111.327639:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342592 00000100:00100000:3.0:1767320111.327686:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342720 00000100:00100000:3.0:1767320111.327691:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342720 00000100:00100000:3.0:1767320111.327711:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3279 00000100:00100000:3.0:1767320111.327717:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985342592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.327766:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985342592:12345-0@lo:400:kworker.0 Request processed in 49us (711us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320111.327771:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3279 00000100:00100000:3.0:1767320111.327784:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3280 00000100:00100000:3.0:1767320111.327790:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985342720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.327819:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985342720:12345-0@lo:400:kworker.0 Request processed in 30us (225us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320111.327825:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3280 00000100:00100000:3.0:1767320111.327878:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985342720:0@lo:400:kworker.0 00000100:00080000:3.0:1767320111.327906:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320111.327910:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320111.327960:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985342336:0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.327988:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320111.328008:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320111.328014:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985342080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.328033:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320111.328058:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342336 00000100:00100000:3.0:1767320111.328061:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342336 00000100:00100000:3.0:1767320111.328093:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342080 00000100:00100000:3.0:1767320111.328096:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342080 00000100:00100000:3.0:1767320111.328115:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5025 00000100:00100000:3.0:1767320111.328121:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985342336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.328156:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985342336:12345-0@lo:400:kworker.0 Request processed in 35us (166us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320111.328203:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5025 00000100:00100000:2.0:1767320111.328777:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5026 00000100:00100000:2.0:1767320111.328797:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985342080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320111.328848:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985342080:12345-0@lo:400:kworker.0 Request processed in 61us (814us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320111.328854:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5026 00000100:00080000:2.0:1767320111.328879:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320111.328896:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985342464:0@lo:38:kworker.0 00000100:00100000:2.0:1767320111.328930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320111.328970:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320111.328976:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320111.328980:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985342208:0@lo:38:kworker.0 00000100:00080000:2.0:1767320111.328995:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320111.328999:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320111.329002:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985341824:0@lo:38:kworker.0 00000100:00100000:2.0:1767320111.329037:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342464 00000100:00100000:2.0:1767320111.329042:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342464 00000100:00100000:2.0:1767320111.329061:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5640 00000100:00100000:2.0:1767320111.329065:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985342464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320111.329085:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320111.329119:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985342464:12345-0@lo:38:kworker.0 Request processed in 51us (188us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320111.329125:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5640 00000100:00080000:2.0:1767320111.329150:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320111.329155:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320111.329159:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985342464:0@lo:38:kworker.0 00000100:00100000:2.0:1767320111.329642:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985342080:0@lo:400:kworker.0 00000100:00100000:2.0:1767320111.329658:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985342336:0@lo:400:kworker.0 00000100:00100000:0.0:1767320111.330384:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985341952 00000100:00100000:0.0:1767320111.330393:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985341952 00000100:00100000:0.0:1767320111.330440:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1900 00000100:00100000:0.0:1767320111.330448:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985341952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320111.330469:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320111.330539:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985341952:12345-0@lo:38:kworker.0 Request processed in 90us (3791us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320111.330547:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1900 00000100:00080000:0.0:1767320111.330606:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320111.330611:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320111.330615:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985341952:0@lo:38:kworker.0 00000100:00100000:1.0:1767320111.330827:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985342592:0@lo:400:kworker.0 00010000:00010000:0.0:1767320111.963568:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320112.987720:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320113.883581:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320113.883618:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985342848:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320113.883693:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320113.885915:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342848 00000100:00100000:2.0:1767320113.885936:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342848 00000100:00100000:2.0:1767320113.886068:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2468 00000100:00100000:2.0:1767320113.886084:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985342848:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320113.886084:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767320113.886140:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320113.886305:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985342848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 226us (2610us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320113.886316:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2468 00000100:00100000:3.0:1767320113.886387:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985342848:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320113.887215:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320113.887245:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985342976:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320113.887311:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320113.887426:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985342976 00000100:00100000:0.0:1767320113.887434:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985342976 00000100:00100000:0.0:1767320113.887503:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2469 00000100:00100000:0.0:1767320113.887512:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985342976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320113.887540:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320113.887587:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985342976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (276us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320113.887594:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2469 00000100:00100000:0.0:1767320113.887646:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985342976:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320113.887809:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320114.011759:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320115.035514:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320115.370878:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767320115.370919:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920598784 offset 0 length 520: 4 00000100:00100000:0.0:1767320115.370947:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320115.371062:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599040 00000100:00100000:2.0:1767320115.371075:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599040 00000100:00100000:3.0:1767320115.371128:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920598912 00000100:00100000:3.0:1767320115.371143:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920598912 00000100:00100000:2.0:1767320115.371238:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5027 00000100:00100000:3.0:1767320115.371253:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5641 00000100:00080000:0.0:1767320115.371257:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320115.371258:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920599040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320115.371271:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920598912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320115.371298:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320115.371451:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920599040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 200us (584us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320115.371462:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5027 00000100:00100000:3.0:1767320115.371982:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920598912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 709us (1028us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320115.371993:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5641 00000100:00100000:0.0:1767320115.419646:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320090/real 1767320090] req@ffff8ce1a5407100 x1853163985336576/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320115 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320115.419696:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a5407100 x1853163985336576/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320115 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320115.419712:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320115.419969:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320115.419975:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985336576:0@lo:250:kworker.0 00000100:00080000:3.0:1767320115.421290:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320115.421300:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320115.421305:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320115.421313:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5339 00000100:00080000:3.0:1767320115.421319:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320115.421328:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320115.421376:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9f80 x1853163985343104/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320115.421407:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320115.421411:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320115.421414:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320115.421417:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5359 00000100:00080000:3.0:1767320115.421420:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320115.421424:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320115.421440:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10700 x1853163985343232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320115.421449:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320115.421452:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320115.421454:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320115.421457:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5359 00000100:00080000:3.0:1767320115.421459:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320115.421462:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320115.421471:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13800 x1853163985343360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320115.421518:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320115.421522:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320115.421523:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320115.421526:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5359 00000100:00080000:3.0:1767320115.421529:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320115.421532:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320115.421543:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2d80 x1853163985343616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320115.421558:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320115.421561:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320115.421563:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320115.421567:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5359 00000100:00080000:3.0:1767320115.421569:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320115.421572:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320115.421580:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0a80 x1853163985343872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320115.421671:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985343488:0@lo:400:kworker.0 00000100:00100000:1.0:1767320115.421758:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320115.421852:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320115.421858:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985343744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320115.421879:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320115.421926:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985343488 00000100:00100000:1.0:1767320115.421935:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985343488 00000100:00100000:1.0:1767320115.421992:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985343744 00000100:00100000:1.0:1767320115.421994:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985343744 00000100:00100000:1.0:1767320115.422015:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5028 00000100:00100000:1.0:1767320115.422023:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985343488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320115.422075:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985343488:12345-0@lo:400:kworker.0 Request processed in 53us (317us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320115.422083:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5028 00000100:00100000:1.0:1767320115.422101:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5029 00000100:00100000:1.0:1767320115.422106:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985343744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320115.422125:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985343744:12345-0@lo:400:kworker.0 Request processed in 19us (245us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320115.422131:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5029 00000100:00080000:1.0:1767320115.422153:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320115.422218:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985343744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320115.422227:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985343488:0@lo:400:kworker.0 00000100:00100000:1.0:1767320115.422280:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985344000:0@lo:400:kworker.0 00000100:00100000:0.0:1767320115.422313:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985343104:0@lo:250:kworker.0 00000100:00100000:1.0:1767320115.422313:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320115.422338:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320115.422345:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985344128:0@lo:400:kworker.0 00000400:00000100:0.0:1767320115.422365:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985343104 offset 0 length 520: 4 00000100:00100000:1.0:1767320115.422372:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320115.422399:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985343232:0@lo:38:kworker.0 00000100:00100000:1.0:1767320115.422413:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344000 00000100:00100000:1.0:1767320115.422418:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344000 00000100:00100000:0.0:1767320115.422433:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320115.422457:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344128 00000100:00100000:0.0:1767320115.422461:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985343360:0@lo:38:kworker.0 00000100:00100000:1.0:1767320115.422462:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344128 00000100:00100000:1.0:1767320115.422489:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3281 00000100:00100000:0.0:1767320115.422499:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320115.422501:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985344000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320115.422519:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985343616:0@lo:38:kworker.0 00000100:00100000:0.0:1767320115.422543:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320115.422558:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985343872:0@lo:38:kworker.0 00000100:00100000:1.0:1767320115.422565:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985344000:12345-0@lo:400:kworker.0 Request processed in 67us (250us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320115.422572:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3281 00000100:00100000:0.0:1767320115.422578:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320115.422588:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3282 00000100:00100000:1.0:1767320115.422596:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985344128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320115.422611:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985343360 00000100:00100000:0.0:1767320115.422615:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985343360 00000100:00100000:1.0:1767320115.422625:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985344128:12345-0@lo:400:kworker.0 Request processed in 30us (252us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320115.422632:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3282 00000100:00100000:0.0:1767320115.422638:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1901 00000100:00100000:0.0:1767320115.422644:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985343360:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767320115.422659:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320115.422663:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00020000:0.0:1767320115.422664:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320115.422680:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985344128:0@lo:400:kworker.0 00000100:00100000:1.0:1767320115.422695:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985344000:0@lo:400:kworker.0 00000100:00100000:0.0:1767320115.422697:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985343360:12345-0@lo:38:kworker.0 Request processed in 52us (200us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320115.422705:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1901 00000100:00100000:1.0:1767320115.422734:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985343232 00000100:00100000:0.0:1767320115.422738:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985343616 00000100:00100000:1.0:1767320115.422739:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985343232 00000100:00100000:0.0:1767320115.422743:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985343616 00000100:00100000:1.0:1767320115.422761:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985343872 00000100:00100000:1.0:1767320115.422766:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985343872 00000100:00100000:0.0:1767320115.422776:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5642 00000100:00100000:0.0:1767320115.422780:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985343232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320115.422788:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320115.422792:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5643 00000100:00100000:1.0:1767320115.422799:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985343616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320115.422811:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320115.422814:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985343232:12345-0@lo:38:kworker.0 Request processed in 31us (381us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320115.422820:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5642 00000100:00100000:0.0:1767320115.422830:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5644 00000100:00100000:0.0:1767320115.422834:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985343872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320115.422841:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320115.422847:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985343616:12345-0@lo:38:kworker.0 Request processed in 46us (302us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320115.422855:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5643 00000100:00100000:0.0:1767320115.422872:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985343872:12345-0@lo:38:kworker.0 Request processed in 27us (285us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320115.422877:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5644 00000100:00080000:0.0:1767320115.422905:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320115.422910:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320115.422913:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985343232:0@lo:38:kworker.0 00000100:00080000:0.0:1767320115.422920:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320115.422924:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320115.422926:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985343360:0@lo:38:kworker.0 00000100:00080000:0.0:1767320115.422932:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320115.422935:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320115.422937:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985343616:0@lo:38:kworker.0 00000100:00080000:0.0:1767320115.422942:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320115.422961:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320115.422965:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985343872:0@lo:38:kworker.0 00010000:00010000:2.0:1767320116.060354:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320117.086945:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320118.110684:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320119.003796:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320119.003830:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985344256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320119.003926:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320119.004401:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985344384:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320119.004447:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320119.006389:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344256 00000100:00100000:1.0:1767320119.006418:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344256 00000100:00100000:1.0:1767320119.006517:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344384 00000100:00100000:1.0:1767320119.006522:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344384 00000100:00080000:2.0:1767320119.006753:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320119.006771:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320119.006813:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2470 00000100:00100000:1.0:1767320119.006826:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985344256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320119.006870:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320119.006998:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2471 00000100:00100000:3.0:1767320119.007008:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985344384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320119.007028:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320119.011305:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985344256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4463us (7360us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320119.011326:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2470 00000100:00100000:3.0:1767320119.011948:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985344384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4937us (7495us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320119.011959:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2471 00000100:00100000:1.0:1767320119.012366:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985344384:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320119.012387:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985344256:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320119.133605:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320120.155530:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320120.492106:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320120.492343:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320120.492465:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599296 00000100:00100000:0.0:1767320120.492475:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599296 00000100:00100000:2.0:1767320120.492627:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599168 00000100:00100000:2.0:1767320120.492640:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599168 00000100:00100000:2.0:1767320120.492725:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5645 00000100:00100000:2.0:1767320120.492735:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920599168:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320120.492767:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320120.492864:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920599168:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 129us (520us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320120.492873:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5645 00000100:00100000:3.0:1767320120.493284:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5030 00000100:00100000:3.0:1767320120.493300:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920599296:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320120.493376:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920599296:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (1289us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320120.493384:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5030 00000100:00080000:1.0:1767320120.496798:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320120.541108:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320120.541125:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320120.541129:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320120.541134:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320120.541140:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320120.541151:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5363 00000100:00080000:3.0:1767320120.541157:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320120.541186:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320120.541243:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985344512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320120.541278:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320120.541282:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320120.541283:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320120.541287:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5363 00000100:00080000:3.0:1767320120.541289:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320120.541293:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320120.541309:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1880 x1853163985344640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320120.541364:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320120.541369:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320120.541371:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320120.541375:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5363 00000100:00080000:3.0:1767320120.541377:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320120.541381:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320120.541393:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2300 x1853163985344896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320120.541410:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320120.541413:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320120.541415:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320120.541419:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5363 00000100:00080000:3.0:1767320120.541421:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320120.541424:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320120.541435:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3480 x1853163985345152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320120.541518:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985345280:0@lo:400:kworker.0 00000100:00100000:3.0:1767320120.541606:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320120.541689:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320120.541696:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985345408:0@lo:400:kworker.0 00000100:00100000:2.0:1767320120.541993:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985345024:0@lo:400:kworker.0 00000100:00100000:2.0:1767320120.542035:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320120.542070:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320120.542076:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985344768:0@lo:400:kworker.0 00000100:00100000:3.0:1767320120.542319:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320120.542356:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345024 00000100:00100000:3.0:1767320120.542366:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345024 00000100:00100000:3.0:1767320120.542431:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5031 00000100:00100000:3.0:1767320120.542442:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985345024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320120.542607:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985344512:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.542642:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320120.542670:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985344640:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.542696:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320120.542741:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985344896:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.542768:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320120.542799:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344512 00000100:00100000:2.0:1767320120.542805:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344512 00000100:00100000:2.0:1767320120.542834:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344896 00000100:00100000:2.0:1767320120.542838:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344896 00000100:00100000:2.0:1767320120.542860:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5646 00000100:00100000:1.0:1767320120.542866:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345280 00000100:00100000:2.0:1767320120.542867:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985344512:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320120.542875:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345280 00010000:00020000:2.0:1767320120.542894:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320120.542911:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345408 00000100:00100000:1.0:1767320120.542915:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345408 00000100:00100000:1.0:1767320120.542944:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3283 00000100:00100000:1.0:1767320120.542952:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985345280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320120.548558:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985344512:12345-0@lo:38:kworker.0 Request processed in 5680us (5904us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320120.548574:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5646 00000100:00100000:2.0:1767320120.548615:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5647 00000100:00100000:2.0:1767320120.548622:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985344896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320120.548645:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320120.548707:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985344896:12345-0@lo:38:kworker.0 Request processed in 84us (5938us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320120.548714:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5647 00000100:00100000:2.0:1767320120.549513:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3284 00000100:00100000:2.0:1767320120.549529:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985345408:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320120.549596:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985345408:12345-0@lo:400:kworker.0 Request processed in 74us (7276us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320120.549603:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3284 00000100:00080000:2.0:1767320120.549626:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320120.549925:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985345408:0@lo:400:kworker.0 00000100:00100000:2.0:1767320120.550105:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344640 00000100:00100000:2.0:1767320120.550112:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344640 00000100:00100000:2.0:1767320120.550146:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1902 00000100:00100000:2.0:1767320120.550151:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985344640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320120.550277:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320120.550315:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985344640:12345-0@lo:38:kworker.0 Request processed in 162us (7618us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320120.550323:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1902 00000100:00100000:2.0:1767320120.550350:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320120.550387:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985344768 00000100:00100000:2.0:1767320120.550392:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985344768 00000100:00100000:2.0:1767320120.550510:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5032 00000100:00100000:2.0:1767320120.550517:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985344768:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320120.550565:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985344768:12345-0@lo:400:kworker.0 Request processed in 49us (214us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320120.550572:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5032 00000100:00080000:2.0:1767320120.550592:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320120.550615:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985344768:0@lo:400:kworker.0 00000100:00100000:1.0:1767320120.550651:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985345280:12345-0@lo:400:kworker.0 Request processed in 7689us (9034us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320120.550667:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3283 00000100:00100000:2.0:1767320120.550724:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985345280:0@lo:400:kworker.0 00000100:00100000:3.0:1767320120.556015:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985345024:12345-0@lo:400:kworker.0 Request processed in 13558us (13963us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320120.556035:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5031 00000100:00080000:0.0:1767320120.556246:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320120.556254:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320120.556259:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985344896:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.556275:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985345152:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.556344:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320120.556413:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320120.556418:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320120.556421:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985344512:0@lo:38:kworker.0 00000100:00100000:3.0:1767320120.556434:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345152 00000100:00080000:0.0:1767320120.556437:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320120.556440:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320120.556442:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345152 00000100:00100000:0.0:1767320120.556443:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985344640:0@lo:38:kworker.0 00000100:00100000:3.0:1767320120.556494:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5648 00000100:00100000:3.0:1767320120.556501:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985345152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320120.556525:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320120.556578:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985345152:12345-0@lo:38:kworker.0 Request processed in 76us (234us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320120.556585:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5648 00000100:00080000:0.0:1767320120.557064:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320120.557070:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320120.557074:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985345152:0@lo:38:kworker.0 00000100:00100000:1.0:1767320120.557947:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985345024:0@lo:400:kworker.0 00010000:00010000:1.0:1767320121.179912:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320122.205684:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320123.227551:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320124.123629:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985345536:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320124.123761:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320124.123944:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345536 00000100:00100000:1.0:1767320124.123955:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345536 00000100:00100000:1.0:1767320124.124048:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2472 00000100:00100000:1.0:1767320124.124058:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985345536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320124.124095:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320124.124153:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985345536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (391us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320124.124161:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2472 00000100:00100000:1.0:1767320124.124260:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985345536:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320124.124547:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320124.125451:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985345664:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320124.125521:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320124.125621:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345664 00000100:00100000:1.0:1767320124.125629:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345664 00000100:00100000:1.0:1767320124.125696:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2473 00000100:00100000:1.0:1767320124.125704:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985345664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320124.125743:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767320124.125810:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320124.126417:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985345664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 711us (894us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320124.126428:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2473 00000100:00100000:3.0:1767320124.126500:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985345664:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320124.252721:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320125.278973:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320125.620780:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320125.620878:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320125.621124:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599552 00000100:00100000:0.0:1767320125.621135:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599552 00000100:00080000:1.0:1767320125.621223:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320125.621271:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599424 00000100:00100000:0.0:1767320125.621276:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599424 00000100:00100000:0.0:1767320125.621312:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5033 00000100:00100000:0.0:1767320125.621327:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920599552:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320125.621413:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920599552:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (643us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320125.621423:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5649 00000100:00100000:1.0:1767320125.621440:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920599424:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320125.621469:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320125.621534:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920599424:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (656us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320125.623437:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5033 00000100:00100000:1.0:1767320125.624354:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5649 00000100:00080000:3.0:1767320125.659317:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.659328:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320125.659331:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.659335:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.659339:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.659348:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.659362:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.659370:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320125.659414:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3480 x1853163985345792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320125.659444:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.659447:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.659449:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.659453:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.659455:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.659458:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320125.659472:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1880 x1853163985345920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320125.659513:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.659517:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.659519:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.659522:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.659525:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.659527:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320125.659536:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2300 x1853163985346176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320125.659551:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.659554:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.659555:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.659559:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.659560:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.659563:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320125.659572:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3100 x1853163985346432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320125.659632:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985346688:0@lo:400:kworker.0 00000100:00100000:3.0:1767320125.659727:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320125.659732:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985346304:0@lo:400:kworker.0 00000100:00100000:3.0:1767320125.659817:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320125.659825:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320125.659831:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985346560:0@lo:400:kworker.0 00000100:00100000:3.0:1767320125.659874:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320125.659907:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320125.659921:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985346048:0@lo:400:kworker.0 00000100:00100000:3.0:1767320125.659953:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346688 00000100:00100000:1.0:1767320125.659963:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320125.659965:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346688 00000100:00100000:3.0:1767320125.660051:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346560 00000100:00100000:1.0:1767320125.660053:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346304 00000100:00100000:3.0:1767320125.660056:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346560 00000100:00100000:1.0:1767320125.660062:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346304 00000100:00100000:3.0:1767320125.660087:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3285 00000100:00100000:3.0:1767320125.660099:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985346688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320125.660128:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346048 00000100:00100000:1.0:1767320125.660133:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346048 00000100:00100000:3.0:1767320125.660200:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985346688:12345-0@lo:400:kworker.0 Request processed in 101us (478us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320125.660208:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3285 00000100:00100000:3.0:1767320125.660226:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3286 00000100:00100000:3.0:1767320125.660233:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985346560:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320125.660261:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985346560:12345-0@lo:400:kworker.0 Request processed in 29us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320125.660267:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3286 00000100:00100000:1.0:1767320125.660303:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5034 00000100:00100000:3.0:1767320125.660311:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985346560:0@lo:400:kworker.0 00000100:00100000:1.0:1767320125.660320:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985346304:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320125.660334:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985346688:0@lo:400:kworker.0 00000100:00080000:3.0:1767320125.660386:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320125.660415:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5035 00000100:00100000:3.0:1767320125.660425:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985346048:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320125.660540:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985345792:0@lo:38:kworker.0 00000100:00100000:3.0:1767320125.663161:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985346048:12345-0@lo:400:kworker.0 Request processed in 2728us (3191us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320125.663194:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5035 00000100:00100000:0.0:1767320125.664129:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985346048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320125.664143:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985346304:0@lo:400:kworker.0 00000100:00100000:1.0:1767320125.670776:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985346304:12345-0@lo:400:kworker.0 Request processed in 10446us (10936us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320125.670800:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5034 00000100:00100000:2.0:1767320125.674194:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320125.676186:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985345920:0@lo:38:kworker.0 00000100:00100000:3.0:1767320125.676257:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320125.676301:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985346176:0@lo:38:kworker.0 00000100:00100000:3.0:1767320125.676328:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320125.676347:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985346432:0@lo:38:kworker.0 00000100:00100000:3.0:1767320125.676366:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320125.676395:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345792 00000100:00100000:3.0:1767320125.676402:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345792 00000100:00100000:3.0:1767320125.676436:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346176 00000100:00100000:3.0:1767320125.676439:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346176 00000100:00100000:3.0:1767320125.676455:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346432 00000100:00100000:3.0:1767320125.676457:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346432 00000100:00100000:3.0:1767320125.676476:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5650 00000100:00100000:3.0:1767320125.676482:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985345792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320125.676503:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320125.676540:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985345792:12345-0@lo:38:kworker.0 Request processed in 57us (2381us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320125.676547:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5650 00000100:00100000:3.0:1767320125.676564:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5651 00000100:00100000:3.0:1767320125.676567:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985346176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320125.676576:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320125.676598:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985346176:12345-0@lo:38:kworker.0 Request processed in 29us (270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320125.676604:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5651 00000100:00100000:3.0:1767320125.676614:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5652 00000100:00100000:3.0:1767320125.676618:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985346432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320125.676626:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320125.676646:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985346432:12345-0@lo:38:kworker.0 Request processed in 27us (280us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320125.676652:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5652 00000100:00080000:3.0:1767320125.676695:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320125.676700:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320125.676704:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985346432:0@lo:38:kworker.0 00000100:00080000:3.0:1767320125.676730:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320125.676734:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320125.676737:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985345792:0@lo:38:kworker.0 00000100:00080000:3.0:1767320125.676743:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320125.676746:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320125.676749:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985346176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320125.677086:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985345920 00000100:00100000:2.0:1767320125.677092:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985345920 00000100:00100000:2.0:1767320125.677117:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1903 00000100:00100000:2.0:1767320125.677122:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985345920:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320125.677132:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320125.677181:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985345920:12345-0@lo:38:kworker.0 Request processed in 57us (924us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320125.677186:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1903 00000100:00080000:3.0:1767320125.677264:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320125.677269:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320125.677272:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985345920:0@lo:38:kworker.0 00010000:00010000:2.0:1767320126.301079:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320127.327177:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320128.348622:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320129.243545:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320129.243570:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985346816:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320129.243639:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320129.243689:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985346944:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320129.243710:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320129.243746:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346816 00000100:00100000:0.0:1767320129.243755:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346816 00000100:00100000:0.0:1767320129.243853:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985346944 00000100:00100000:0.0:1767320129.243857:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985346944 00000100:00080000:3.0:1767320129.243883:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320129.243887:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320129.243913:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2474 00000100:00100000:1.0:1767320129.243925:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985346816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320129.243962:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320129.243989:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2475 00000100:00100000:0.0:1767320129.244000:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985346944:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320129.244016:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985346816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (375us total) trans 0 rc 0/0 00002000:00000024:0.0:1767320129.244020:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320129.244027:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2474 00000100:00100000:0.0:1767320129.244060:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985346944:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (348us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320129.244069:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2475 00000100:00100000:1.0:1767320129.244082:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985346944:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320129.244100:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985346816:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320129.371410:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320130.397291:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320130.732650:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320130.732721:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599808 00000100:00100000:2.0:1767320130.732731:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599808 00000100:00100000:0.0:1767320130.732740:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320130.732789:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599680 00000100:00100000:1.0:1767320130.732795:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599680 00000100:00100000:1.0:1767320130.732959:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5036 00000100:00100000:1.0:1767320130.732972:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920599808:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320130.733062:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920599808:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (418us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320130.733071:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5036 00000100:00100000:1.0:1767320130.733090:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5653 00000100:00100000:1.0:1767320130.733094:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920599680:12345-192.168.204.26@tcp:38:kworker.0 00000100:00080000:0.0:1767320130.733114:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:1.0:1767320130.733115:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320130.733145:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920599680:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (405us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320130.733152:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5653 00000100:00080000:3.0:1767320130.779328:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320130.779338:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320130.779341:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320130.779345:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320130.779349:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320130.779357:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5373 00000100:00080000:3.0:1767320130.779362:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320130.779368:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320130.779413:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519500 x1853163985347072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320130.779430:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320130.779433:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320130.779435:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320130.779438:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5373 00000100:00080000:3.0:1767320130.779440:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320130.779443:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320130.779455:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518000 x1853163985347200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320130.779498:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320130.779501:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320130.779503:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320130.779506:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5373 00000100:00080000:3.0:1767320130.779509:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320130.779512:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320130.779524:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351aa00 x1853163985347456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320130.779539:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320130.779543:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320130.779545:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320130.779549:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5373 00000100:00080000:3.0:1767320130.779552:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320130.779555:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320130.779565:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519180 x1853163985347712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320130.779573:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985347328:0@lo:400:kworker.0 00000100:00100000:3.0:1767320130.779617:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985347968:0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.779653:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320130.779659:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320130.779724:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320130.779731:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985347584:0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.779756:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320130.779763:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985347072:0@lo:38:kworker.0 00000100:00100000:2.0:1767320130.779795:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985347840:0@lo:400:kworker.0 00000100:00100000:0.0:1767320130.779802:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320130.779831:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985347200:0@lo:38:kworker.0 00000100:00100000:2.0:1767320130.779832:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320130.779865:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320130.779882:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347328 00000100:00100000:2.0:1767320130.779891:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347328 00000100:00100000:0.0:1767320130.779897:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985347456:0@lo:38:kworker.0 00000100:00100000:0.0:1767320130.779929:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320130.779950:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985347712:0@lo:38:kworker.0 00000100:00100000:2.0:1767320130.779962:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347584 00000100:00100000:2.0:1767320130.779968:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347584 00000100:00100000:1.0:1767320130.779969:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347200 00000100:00100000:0.0:1767320130.779972:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320130.779974:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347200 00000100:00100000:2.0:1767320130.779989:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347072 00000100:00100000:2.0:1767320130.779994:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347072 00000100:00080000:0.0:1767320130.780005:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320130.780007:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347456 00000100:00100000:2.0:1767320130.780011:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347456 00000100:00100000:1.0:1767320130.780017:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1904 00000100:00100000:0.0:1767320130.780018:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347712 00000100:00100000:0.0:1767320130.780021:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347712 00000100:00100000:1.0:1767320130.780026:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985347200:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320130.780029:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5037 00000100:00100000:2.0:1767320130.780041:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985347328:12345-0@lo:400:kworker.0 00010000:00020000:1.0:1767320130.780049:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320130.780057:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5038 00000100:00100000:0.0:1767320130.780065:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985347584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320130.780097:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985347200:12345-0@lo:38:kworker.0 Request processed in 72us (232us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320130.780104:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1904 00000100:00100000:2.0:1767320130.780132:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985347328:12345-0@lo:400:kworker.0 Request processed in 91us (479us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320130.780134:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5654 00000100:00100000:1.0:1767320130.780142:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985347072:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320130.780147:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5037 00010000:00020000:1.0:1767320130.780156:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320130.780187:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5655 00000100:00100000:1.0:1767320130.780207:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985347072:12345-0@lo:38:kworker.0 Request processed in 64us (407us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320130.780214:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5654 00000100:00100000:1.0:1767320130.780228:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5656 00000100:00100000:1.0:1767320130.780232:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985347712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320130.780240:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320130.780262:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985347712:12345-0@lo:38:kworker.0 Request processed in 29us (290us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320130.780267:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5656 00000100:00080000:1.0:1767320130.780297:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320130.780302:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320130.780305:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985347072:0@lo:38:kworker.0 00000100:00080000:1.0:1767320130.780313:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320130.780315:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320130.780318:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985347200:0@lo:38:kworker.0 00000100:00080000:1.0:1767320130.780324:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320130.780327:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320130.780330:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985347712:0@lo:38:kworker.0 00000100:00100000:1.0:1767320130.780380:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347968 00000100:00100000:1.0:1767320130.780384:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347968 00000100:00100000:1.0:1767320130.780408:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985347840 00000100:00100000:1.0:1767320130.780411:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985347840 00000100:00100000:2.0:1767320130.780417:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985347456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320130.780431:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320130.780442:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3287 00000100:00100000:1.0:1767320130.780450:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985347968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.780468:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985347456:12345-0@lo:38:kworker.0 Request processed in 273us (536us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320130.780476:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5655 00000100:00100000:1.0:1767320130.780488:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985347968:12345-0@lo:400:kworker.0 Request processed in 38us (829us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320130.780495:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3287 00000100:00100000:1.0:1767320130.780507:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3288 00000100:00100000:1.0:1767320130.780513:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985347840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320130.780543:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985347840:12345-0@lo:400:kworker.0 Request processed in 30us (713us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320130.780551:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3288 00000100:00100000:1.0:1767320130.780579:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985347840:0@lo:400:kworker.0 00000100:00080000:1.0:1767320130.780597:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320130.780600:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320130.780602:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985347456:0@lo:38:kworker.0 00000100:00100000:3.0:1767320130.783842:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985347968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320130.784109:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985347584:12345-0@lo:400:kworker.0 Request processed in 4042us (4349us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320130.784120:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5038 00000100:00100000:2.0:1767320130.785773:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985347584:0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.785789:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985347328:0@lo:400:kworker.0 00010000:00010000:1.0:1767320131.419412:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320132.443676:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320133.474910:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320134.363613:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985348224:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320134.363688:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320134.363763:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320134.363769:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985348096:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320134.363793:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320134.363871:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348224 00000100:00100000:0.0:1767320134.363878:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348224 00000100:00100000:0.0:1767320134.363935:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348096 00000100:00100000:0.0:1767320134.363938:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348096 00000100:00100000:0.0:1767320134.363958:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2476 00000100:00100000:0.0:1767320134.363967:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985348224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320134.364000:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320134.364030:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2477 00000100:00100000:2.0:1767320134.364040:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985348096:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320134.364055:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985348224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (367us total) trans 0 rc 0/0 00002000:00000024:2.0:1767320134.364056:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320134.364063:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2476 00000100:00100000:2.0:1767320134.364082:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985348096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (288us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320134.364087:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2477 00000100:00100000:0.0:1767320134.364105:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985348096:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320134.364118:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985348224:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320134.364457:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320134.364462:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320134.492434:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320135.519265:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320135.850570:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320135.850674:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920599936 00000100:00100000:1.0:1767320135.850684:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920599936 00000100:00100000:1.0:1767320135.850751:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5657 00000100:00100000:1.0:1767320135.850759:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920599936:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320135.850781:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320135.850858:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920599936:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 97us (296us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320135.850865:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5657 00000100:00100000:0.0:1767320135.856230:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320135.856354:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920600064 00000100:00100000:1.0:1767320135.856363:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920600064 00000100:00080000:3.0:1767320135.856415:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320135.856458:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5039 00000100:00100000:1.0:1767320135.856467:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920600064:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320135.856530:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920600064:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 65us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320135.856535:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5039 00000100:00080000:3.0:1767320135.899273:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320135.899284:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320135.899289:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320135.899293:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320135.899297:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320135.899307:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5379 00000100:00080000:3.0:1767320135.899312:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320135.899318:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320135.899356:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519c00 x1853163985348352/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320135.899397:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320135.899401:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320135.899403:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320135.899407:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5379 00000100:00080000:3.0:1767320135.899409:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320135.899412:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320135.899426:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518380 x1853163985348480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320135.899465:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320135.899469:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320135.899471:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320135.899475:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5379 00000100:00080000:3.0:1767320135.899477:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320135.899480:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320135.899493:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b800 x1853163985348736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320135.899511:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320135.899514:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320135.899515:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320135.899519:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5379 00000100:00080000:3.0:1767320135.899521:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320135.899524:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320135.899534:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518000 x1853163985348992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320135.902123:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985348352:0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.902230:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320135.902308:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985348480:0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.902336:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320135.902362:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985348736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.902385:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320135.902404:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985348992:0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.902424:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320135.902462:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348480 00000100:00100000:1.0:1767320135.902470:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348480 00000100:00100000:1.0:1767320135.902511:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1905 00000100:00100000:1.0:1767320135.902518:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985348480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320135.902538:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320135.902578:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985348480:12345-0@lo:38:kworker.0 Request processed in 59us (241us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320135.902586:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1905 00000100:00100000:1.0:1767320135.902610:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348352 00000100:00100000:1.0:1767320135.902613:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348352 00000100:00100000:1.0:1767320135.902630:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348736 00000100:00100000:1.0:1767320135.902632:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348736 00000100:00100000:1.0:1767320135.902642:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348992 00000100:00100000:1.0:1767320135.902644:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348992 00000100:00100000:1.0:1767320135.902657:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5658 00000100:00100000:1.0:1767320135.902662:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985348352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320135.902671:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320135.902694:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985348352:12345-0@lo:38:kworker.0 Request processed in 31us (465us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320135.902699:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5658 00000100:00100000:1.0:1767320135.902724:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5659 00000100:00100000:1.0:1767320135.902729:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985348736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320135.902737:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320135.902866:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5660 00000100:00100000:3.0:1767320135.902872:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985348992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320135.902882:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320135.902911:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985348992:12345-0@lo:38:kworker.0 Request processed in 38us (486us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320135.902916:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5660 00000100:00080000:3.0:1767320135.903005:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.903011:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.903015:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985348352:0@lo:38:kworker.0 00000100:00080000:3.0:1767320135.903025:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.903028:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.903031:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985348480:0@lo:38:kworker.0 00000100:00080000:3.0:1767320135.903039:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.903042:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.903045:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985348992:0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.903135:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985348736:12345-0@lo:38:kworker.0 Request processed in 403us (749us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320135.903141:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5659 00000100:00080000:3.0:1767320135.903209:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.903213:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.903216:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985348736:0@lo:38:kworker.0 00000100:00100000:0.0:1767320135.903298:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985348864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.903328:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320135.903355:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320135.903362:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985348608:0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.903382:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320135.903415:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985349248:0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.903440:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320135.903460:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320135.903467:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985349120:0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.903490:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320135.903515:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985349248 00000100:00100000:0.0:1767320135.903520:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985349248 00000100:00100000:0.0:1767320135.903549:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985349120 00000100:00100000:0.0:1767320135.903552:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985349120 00000100:00100000:0.0:1767320135.903568:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3289 00000100:00100000:0.0:1767320135.903574:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985349248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.903615:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985349248:12345-0@lo:400:kworker.0 Request processed in 41us (174us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320135.903622:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3289 00000100:00100000:0.0:1767320135.903637:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3290 00000100:00100000:0.0:1767320135.903643:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985349120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.903671:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985349120:12345-0@lo:400:kworker.0 Request processed in 28us (179us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320135.903676:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3290 00000100:00100000:0.0:1767320135.903703:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985349120:0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.903712:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985349248:0@lo:400:kworker.0 00000100:00100000:3.0:1767320135.903855:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348864 00000100:00100000:3.0:1767320135.903860:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348864 00000100:00100000:3.0:1767320135.903903:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985348608 00000100:00100000:3.0:1767320135.903907:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985348608 00000100:00100000:3.0:1767320135.903928:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5040 00000100:00100000:3.0:1767320135.903933:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985348864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320135.903968:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985348864:12345-0@lo:400:kworker.0 Request processed in 35us (639us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320135.903974:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5040 00000100:00100000:3.0:1767320135.903986:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5041 00000100:00100000:3.0:1767320135.903993:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985348608:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320135.904017:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985348608:12345-0@lo:400:kworker.0 Request processed in 25us (634us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320135.904022:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5041 00000100:00080000:0.0:1767320135.904951:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320135.904988:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985348608:0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.905000:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985348864:0@lo:400:kworker.0 00010000:00010000:0.0:1767320136.539389:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320137.565236:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320138.588538:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320139.483966:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320139.484001:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985349376:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320139.484101:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320139.484262:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985349376 00000100:00100000:0.0:1767320139.484271:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985349376 00000100:00100000:0.0:1767320139.484344:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2478 00000100:00100000:0.0:1767320139.484355:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985349376:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320139.484395:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320139.484463:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985349376:12345-0@lo:13:osp-pre-0-1.0 Request processed in 110us (362us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320139.484473:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2478 00000100:00080000:0.0:1767320139.484498:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320139.484542:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985349376:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320139.493815:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320139.493846:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985349504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320139.493937:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320139.494157:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985349504 00000100:00100000:1.0:1767320139.494192:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985349504 00000100:00100000:1.0:1767320139.494265:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2479 00000100:00100000:1.0:1767320139.494276:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985349504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320139.494315:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767320139.494320:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320139.494597:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985349504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 322us (659us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320139.494607:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2479 00000100:00100000:3.0:1767320139.499285:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985349504:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320139.612917:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320140.635648:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767320140.970241:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920600192 offset 0 length 520: 4 00000100:00100000:3.0:1767320140.970308:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320140.970366:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320140.970498:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920600448 00000100:00100000:1.0:1767320140.970513:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920600448 00000100:00100000:1.0:1767320140.970607:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920600320 00000100:00100000:1.0:1767320140.970611:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920600320 00000100:00100000:1.0:1767320140.970635:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5042 00000100:00100000:1.0:1767320140.970648:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920600448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320140.970784:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920600448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 137us (474us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320140.970793:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5042 00000100:00100000:1.0:1767320140.970813:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5661 00000100:00100000:1.0:1767320140.970818:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920600320:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320140.970847:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320140.970877:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920600320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 57us (511us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320140.970884:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5661 00000100:00080000:2.0:1767320140.971568:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320141.020278:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320115/real 1767320115] req@ffff8ce1b5be9f80 x1853163985343104/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320140 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320141.020330:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5be9f80 x1853163985343104/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320140 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320141.020353:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320141.020363:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320141.020368:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985343104:0@lo:250:kworker.0 00000100:00080000:2.0:1767320141.020746:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320141.020753:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320141.020756:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320141.020763:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5363 00000100:00080000:2.0:1767320141.020769:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320141.020776:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320141.020817:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7b80 x1853163985349632/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320141.020845:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320141.020849:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320141.020851:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320141.020855:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5384 00000100:00080000:2.0:1767320141.020857:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320141.020862:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320141.020877:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985349632:0@lo:250:kworker.0 00000100:00100000:2.0:1767320141.020878:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7100 x1853163985349760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320141.020888:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320141.020891:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320141.020892:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320141.020895:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5384 00000100:00080000:2.0:1767320141.020897:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320141.020900:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320141.020911:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985349888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:3.0:1767320141.020937:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985349632 offset 0 length 520: 4 00000100:00080000:2.0:1767320141.020953:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320141.020956:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320141.020958:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320141.020962:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5384 00000100:00080000:2.0:1767320141.020965:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320141.020968:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320141.020978:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985350144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320141.020993:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320141.020996:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320141.020998:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320141.021001:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5384 00000100:00080000:2.0:1767320141.021003:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320141.021007:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320141.021018:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985350400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320141.021075:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985350528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320141.021119:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320141.021127:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985349760:0@lo:38:kworker.0 00000100:00100000:2.0:1767320141.021146:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320141.021152:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985350656:0@lo:400:kworker.0 00000100:00100000:3.0:1767320141.021156:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320141.021241:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985350016:0@lo:400:kworker.0 00000100:00100000:1.0:1767320141.021321:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985350272:0@lo:400:kworker.0 00000100:00100000:3.0:1767320141.021657:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985349888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320141.021687:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320141.021706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985350144:0@lo:38:kworker.0 00000100:00100000:3.0:1767320141.021735:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320141.021751:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985350400:0@lo:38:kworker.0 00000100:00100000:3.0:1767320141.021769:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320141.021813:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985349760 00000100:00100000:3.0:1767320141.021821:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985349760 00000100:00100000:3.0:1767320141.021861:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350144 00000100:00100000:3.0:1767320141.021864:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350144 00000100:00100000:3.0:1767320141.021879:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350400 00000100:00100000:3.0:1767320141.021882:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350400 00000100:00100000:3.0:1767320141.021901:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5662 00000100:00100000:3.0:1767320141.021910:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985349760:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320141.021936:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320141.022244:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985349760:12345-0@lo:38:kworker.0 Request processed in 75us (827us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320141.022253:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5662 00000100:00100000:3.0:1767320141.022274:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5663 00000100:00100000:3.0:1767320141.022277:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985350144:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320141.022286:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320141.022376:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985350144:12345-0@lo:38:kworker.0 Request processed in 35us (578us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320141.022383:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5663 00000100:00100000:3.0:1767320141.022398:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5664 00000100:00100000:3.0:1767320141.022402:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985350400:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320141.022411:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320141.022436:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985350400:12345-0@lo:38:kworker.0 Request processed in 32us (666us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320141.022441:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5664 00000100:00100000:3.0:1767320141.022464:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985349888 00000100:00100000:3.0:1767320141.022468:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985349888 00000100:00100000:3.0:1767320141.022492:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1906 00000100:00100000:3.0:1767320141.022496:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985349888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320141.022505:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320141.022530:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985349888:12345-0@lo:38:kworker.0 Request processed in 32us (842us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320141.022535:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1906 00000100:00080000:3.0:1767320141.022595:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320141.022600:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320141.022604:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985349760:0@lo:38:kworker.0 00000100:00080000:3.0:1767320141.022614:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320141.022617:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320141.022620:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985349888:0@lo:38:kworker.0 00000100:00080000:3.0:1767320141.022628:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320141.022632:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320141.022634:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985350144:0@lo:38:kworker.0 00000100:00080000:3.0:1767320141.022642:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320141.022645:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320141.022648:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985350400:0@lo:38:kworker.0 00000100:00100000:3.0:1767320141.022727:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350528 00000100:00100000:3.0:1767320141.022731:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350528 00000100:00100000:3.0:1767320141.022785:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3291 00000100:00100000:3.0:1767320141.022791:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985350528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320141.022836:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985350528:12345-0@lo:400:kworker.0 Request processed in 45us (1717us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320141.022842:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3291 00000100:00080000:3.0:1767320141.022871:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320141.023909:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320141.023972:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985350528:0@lo:400:kworker.0 00000100:00100000:0.0:1767320141.024139:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320141.024358:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350016 00000100:00100000:2.0:1767320141.024364:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350016 00000100:00080000:0.0:1767320141.024420:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320141.024423:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5043 00000100:00100000:2.0:1767320141.024433:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985350016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320141.024487:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985350016:12345-0@lo:400:kworker.0 Request processed in 57us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320141.024495:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5043 00000100:00100000:0.0:1767320141.024516:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985350016:0@lo:400:kworker.0 00000100:00100000:3.0:1767320141.025151:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350656 00000100:00100000:3.0:1767320141.025157:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350656 00000100:00080000:0.0:1767320141.025209:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320141.025228:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3292 00000100:00100000:3.0:1767320141.025237:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985350656:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320141.025289:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985350656:12345-0@lo:400:kworker.0 Request processed in 53us (1384us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320141.025296:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3292 00000100:00100000:0.0:1767320141.025321:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985350656:0@lo:400:kworker.0 00000100:00100000:1.0:1767320141.025808:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320141.025940:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350272 00000100:00100000:2.0:1767320141.025946:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350272 00000100:00100000:2.0:1767320141.026058:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5044 00000100:00100000:2.0:1767320141.026067:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985350272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320141.026116:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985350272:12345-0@lo:400:kworker.0 Request processed in 51us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320141.026124:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5044 00000100:00100000:1.0:1767320141.026200:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985350272:0@lo:400:kworker.0 00000100:00080000:0.0:1767320141.026454:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320141.659854:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320142.684661:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320143.707481:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320144.604351:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320144.604396:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985350784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320144.604492:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320144.604643:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350784 00000100:00100000:0.0:1767320144.604683:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350784 00000100:00100000:0.0:1767320144.604778:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2480 00000100:00100000:0.0:1767320144.604790:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985350784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320144.604838:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320144.604894:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985350784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 107us (401us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320144.604903:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2480 00000100:00080000:1.0:1767320144.605921:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320144.605998:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985350784:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320144.606834:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985350912:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320144.606911:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320144.607012:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985350912 00000100:00100000:0.0:1767320144.607022:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985350912 00000100:00100000:0.0:1767320144.607098:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2481 00000100:00080000:3.0:1767320144.607106:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320144.607110:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985350912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320144.607192:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320144.612932:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985350912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 5810us (6004us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320144.612956:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2481 00000100:00100000:2.0:1767320144.613680:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985350912:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320144.732388:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320145.755445:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320146.091461:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320146.091918:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320146.098039:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920600704 00000100:00100000:2.0:1767320146.098061:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920600704 00000100:00080000:1.0:1767320146.098223:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320146.098279:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920600576 00000100:00100000:2.0:1767320146.098283:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920600576 00000100:00100000:3.0:1767320146.098343:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5045 00000100:00100000:3.0:1767320146.098357:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920600704:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320146.098473:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920600704:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 119us (7027us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320146.098481:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5045 00000100:00100000:3.0:1767320146.098516:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5665 00000100:00100000:3.0:1767320146.098522:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920600576:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320146.098547:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320146.098593:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920600576:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 68us (6680us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320146.098600:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5665 00000100:00080000:2.0:1767320146.145420:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320146.145434:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320146.145437:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320146.145443:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320146.145448:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320146.145460:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5389 00000100:00080000:2.0:1767320146.145466:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320146.145482:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320146.145540:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe23100 x1853163985351040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320146.145575:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320146.145579:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320146.145582:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320146.145586:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5389 00000100:00080000:2.0:1767320146.145588:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320146.145592:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320146.145617:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985351168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320146.145691:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320146.145694:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320146.145696:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320146.145700:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5389 00000100:00080000:2.0:1767320146.145703:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320146.145707:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320146.145722:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22d80 x1853163985351424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320146.145737:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320146.145741:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320146.145743:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320146.145746:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5389 00000100:00080000:2.0:1767320146.145748:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320146.145751:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320146.145759:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20a80 x1853163985351680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320146.145810:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985351808:0@lo:400:kworker.0 00000100:00100000:2.0:1767320146.145893:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320146.145975:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320146.145981:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985351936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320146.146005:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320146.146043:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351808 00000100:00100000:2.0:1767320146.146051:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351808 00000100:00100000:2.0:1767320146.146113:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351936 00000100:00100000:2.0:1767320146.146117:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351936 00000100:00100000:2.0:1767320146.146142:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3293 00000100:00100000:2.0:1767320146.146151:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985351808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320146.146230:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985351808:12345-0@lo:400:kworker.0 Request processed in 79us (336us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320146.146237:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3293 00000100:00100000:2.0:1767320146.146255:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3294 00000100:00100000:2.0:1767320146.146261:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985351936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320146.146282:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985351936:12345-0@lo:400:kworker.0 Request processed in 22us (276us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320146.146287:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3294 00000100:00100000:2.0:1767320146.146328:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985351936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320146.146337:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985351808:0@lo:400:kworker.0 00000100:00100000:1.0:1767320146.146632:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985351552:0@lo:400:kworker.0 00000100:00100000:1.0:1767320146.146670:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320146.146710:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320146.146716:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985351296:0@lo:400:kworker.0 00000100:00100000:1.0:1767320146.146740:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320146.146779:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320146.146784:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320146.146797:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351552 00000100:00100000:1.0:1767320146.146801:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351552 00000100:00100000:1.0:1767320146.146838:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351296 00000100:00100000:1.0:1767320146.146841:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351296 00000100:00100000:1.0:1767320146.146865:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5046 00000100:00100000:1.0:1767320146.146871:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985351552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320146.146925:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985351552:12345-0@lo:400:kworker.0 Request processed in 53us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320146.146930:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5046 00000100:00100000:1.0:1767320146.146943:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5047 00000100:00100000:1.0:1767320146.146949:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985351296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320146.146976:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985351296:12345-0@lo:400:kworker.0 Request processed in 27us (235us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320146.146982:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5047 00000100:00080000:1.0:1767320146.147003:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320146.147023:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985351296:0@lo:400:kworker.0 00000100:00100000:1.0:1767320146.147032:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985351552:0@lo:400:kworker.0 00000100:00100000:3.0:1767320146.147347:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985351040:0@lo:38:kworker.0 00000100:00100000:3.0:1767320146.147397:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320146.147426:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985351168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320146.147453:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320146.147487:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985351424:0@lo:38:kworker.0 00000100:00100000:3.0:1767320146.147511:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320146.147529:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985351680:0@lo:38:kworker.0 00000100:00100000:3.0:1767320146.147550:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320146.147583:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351168 00000100:00100000:3.0:1767320146.147587:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351168 00000100:00100000:3.0:1767320146.147633:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1907 00000100:00100000:3.0:1767320146.147639:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985351168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320146.147661:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320146.147697:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985351168:12345-0@lo:38:kworker.0 Request processed in 57us (243us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320146.147704:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1907 00000100:00080000:3.0:1767320146.147739:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320146.147745:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320146.147748:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985351168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320146.147770:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351040 00000100:00100000:3.0:1767320146.147773:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351040 00000100:00100000:3.0:1767320146.147792:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351424 00000100:00100000:3.0:1767320146.147795:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351424 00000100:00100000:3.0:1767320146.147807:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985351680 00000100:00100000:3.0:1767320146.147809:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985351680 00000100:00100000:3.0:1767320146.147818:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5666 00000100:00100000:3.0:1767320146.147822:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985351040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320146.147831:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320146.147859:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985351040:12345-0@lo:38:kworker.0 Request processed in 36us (462us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320146.147866:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5666 00000100:00100000:3.0:1767320146.147876:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5667 00000100:00100000:3.0:1767320146.147880:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985351424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320146.147888:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320146.147921:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985351424:12345-0@lo:38:kworker.0 Request processed in 39us (409us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320146.147927:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5667 00000100:00100000:3.0:1767320146.147937:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5668 00000100:00100000:3.0:1767320146.147940:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985351680:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320146.147948:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320146.147972:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985351680:12345-0@lo:38:kworker.0 Request processed in 31us (421us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320146.147978:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5668 00000100:00080000:3.0:1767320146.148006:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320146.148009:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320146.148012:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985351040:0@lo:38:kworker.0 00000100:00080000:3.0:1767320146.148020:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320146.148024:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320146.148027:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985351424:0@lo:38:kworker.0 00000100:00080000:3.0:1767320146.148033:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320146.148036:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320146.148038:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985351680:0@lo:38:kworker.0 00010000:00010000:2.0:1767320146.779442:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320147.803824:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320148.833131:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320149.723602:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320149.723627:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985352064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320149.723700:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320149.724036:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352064 00000100:00100000:1.0:1767320149.724044:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352064 00000100:00100000:1.0:1767320149.724108:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2482 00000100:00100000:1.0:1767320149.724118:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985352064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320149.724147:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320149.724213:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985352064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (513us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320149.724221:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2482 00000100:00080000:3.0:1767320149.724280:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320149.724320:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985352064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320149.724588:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985352192:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320149.724618:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320149.724720:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352192 00000100:00100000:1.0:1767320149.724725:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352192 00000100:00100000:1.0:1767320149.724780:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2483 00000100:00100000:1.0:1767320149.724787:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985352192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320149.724808:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767320149.724863:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320149.726287:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985352192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1494us (1662us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320149.726300:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2483 00000100:00100000:3.0:1767320149.726658:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985352192:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320149.851879:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320150.875585:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320151.211559:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320151.211606:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320151.211694:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920600960 00000100:00100000:2.0:1767320151.211704:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920600960 00000100:00100000:2.0:1767320151.211770:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920600832 00000100:00100000:2.0:1767320151.211773:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920600832 00000100:00100000:2.0:1767320151.211786:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5048 00000100:00100000:2.0:1767320151.211794:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920600960:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320151.211857:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920600960:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 64us (309us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320151.211864:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5048 00000100:00100000:2.0:1767320151.211882:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5669 00000100:00100000:2.0:1767320151.211886:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920600832:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320151.211906:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320151.211930:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920600832:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (324us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320151.211936:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5669 00000100:00080000:1.0:1767320151.211990:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320151.259439:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320151.259449:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320151.259452:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320151.259456:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320151.259461:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320151.259470:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320151.259474:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320151.259480:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320151.259519:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7100 x1853163985352320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320151.259561:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320151.259565:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320151.259566:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320151.259570:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320151.259572:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320151.259576:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320151.259589:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985352448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320151.259625:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320151.259628:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320151.259631:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320151.259634:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320151.259636:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320151.259639:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320151.259648:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8700 x1853163985352704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320151.259662:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320151.259665:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320151.259666:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320151.259670:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320151.259672:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320151.259675:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320151.259684:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8a80 x1853163985352960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320151.259735:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985353088:0@lo:400:kworker.0 00000100:00100000:3.0:1767320151.259804:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320151.259869:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320151.259875:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985353216:0@lo:400:kworker.0 00000100:00100000:3.0:1767320151.259897:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320151.259928:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353088 00000100:00100000:3.0:1767320151.259934:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353088 00000100:00100000:3.0:1767320151.259979:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353216 00000100:00100000:3.0:1767320151.259982:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353216 00000100:00100000:3.0:1767320151.260002:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3295 00000100:00100000:3.0:1767320151.260010:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a04850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985353088:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320151.260057:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a04850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985353088:12345-0@lo:400:kworker.0 Request processed in 47us (252us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320151.260063:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3295 00000100:00100000:3.0:1767320151.260273:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3296 00000100:00100000:3.0:1767320151.260280:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985353216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320151.260307:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985353216:12345-0@lo:400:kworker.0 Request processed in 28us (408us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320151.260313:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3296 00000100:00100000:3.0:1767320151.260346:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985353216:0@lo:400:kworker.0 00000100:00100000:3.0:1767320151.260355:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985353088:0@lo:400:kworker.0 00000100:00080000:1.0:1767320151.260424:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320151.260429:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320151.260440:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985352320:0@lo:38:kworker.0 00000100:00100000:1.0:1767320151.260475:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320151.260507:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985352448:0@lo:38:kworker.0 00000100:00100000:1.0:1767320151.260544:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320151.260568:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985352704:0@lo:38:kworker.0 00000100:00100000:1.0:1767320151.260588:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320151.260604:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985352960:0@lo:38:kworker.0 00000100:00100000:1.0:1767320151.260617:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320151.260639:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352448 00000100:00100000:1.0:1767320151.260641:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352448 00000100:00100000:1.0:1767320151.260658:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1908 00000100:00100000:1.0:1767320151.260661:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985352448:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320151.260677:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320151.260708:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985352448:12345-0@lo:38:kworker.0 Request processed in 44us (163us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320151.260714:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1908 00000100:00100000:1.0:1767320151.260737:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352320 00000100:00100000:1.0:1767320151.260740:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352320 00000100:00100000:1.0:1767320151.260755:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352704 00000100:00100000:1.0:1767320151.260758:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352704 00000100:00100000:1.0:1767320151.260768:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352960 00000100:00100000:1.0:1767320151.260770:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352960 00000100:00100000:1.0:1767320151.260782:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5670 00000100:00100000:1.0:1767320151.260786:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985352320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320151.260796:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320151.260820:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985352320:12345-0@lo:38:kworker.0 Request processed in 32us (344us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320151.260825:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5670 00000100:00100000:1.0:1767320151.260832:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5671 00000100:00100000:1.0:1767320151.260835:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985352704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320151.260841:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320151.260858:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985352704:12345-0@lo:38:kworker.0 Request processed in 21us (270us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320151.260863:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5671 00000100:00100000:1.0:1767320151.260872:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5672 00000100:00100000:1.0:1767320151.260876:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985352960:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320151.260884:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320151.260904:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985352960:12345-0@lo:38:kworker.0 Request processed in 27us (285us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320151.260910:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5672 00000100:00080000:1.0:1767320151.260936:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320151.260940:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320151.260943:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985352320:0@lo:38:kworker.0 00000100:00080000:1.0:1767320151.260951:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320151.260954:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320151.260957:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985352448:0@lo:38:kworker.0 00000100:00080000:1.0:1767320151.260963:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320151.260965:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320151.260968:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985352704:0@lo:38:kworker.0 00000100:00080000:1.0:1767320151.260975:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320151.260978:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320151.260980:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985352960:0@lo:38:kworker.0 00000100:00100000:2.0:1767320151.261054:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985352576:0@lo:400:kworker.0 00000100:00100000:2.0:1767320151.261082:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320151.261104:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320151.261108:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985352832:0@lo:400:kworker.0 00000100:00100000:2.0:1767320151.261121:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320151.261143:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352576 00000100:00100000:2.0:1767320151.261145:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352576 00000100:00100000:2.0:1767320151.261188:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5049 00000100:00100000:2.0:1767320151.261194:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985352576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320151.261227:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985352576:12345-0@lo:400:kworker.0 Request processed in 34us (144us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320151.261233:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5049 00000100:00100000:2.0:1767320151.261244:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985352832 00000100:00100000:2.0:1767320151.261247:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985352832 00000100:00100000:2.0:1767320151.261262:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5050 00000100:00100000:2.0:1767320151.261266:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985352832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320151.261290:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985352832:12345-0@lo:400:kworker.0 Request processed in 23us (167us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320151.261295:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5050 00000100:00080000:2.0:1767320151.261307:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320151.261324:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985352832:0@lo:400:kworker.0 00000100:00100000:2.0:1767320151.261333:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985352576:0@lo:400:kworker.0 00010000:00010000:1.0:1767320151.899394:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320152.923551:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320153.947403:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320154.843596:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985353344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320154.843670:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320154.843965:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353344 00000100:00100000:1.0:1767320154.843973:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353344 00000100:00100000:1.0:1767320154.844044:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2484 00000100:00100000:1.0:1767320154.844054:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985353344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320154.844084:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320154.844131:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985353344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (461us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320154.844138:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2484 00000100:00080000:2.0:1767320154.844202:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320154.844238:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985353344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320154.844481:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320154.844490:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985353472:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320154.844517:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320154.844762:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353472 00000100:00100000:0.0:1767320154.844767:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353472 00000100:00100000:0.0:1767320154.844808:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2485 00000100:00100000:0.0:1767320154.844816:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985353472:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320154.844833:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320154.844879:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985353472:12345-0@lo:13:osp-pre-0-1.0 Request processed in 64us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320154.844886:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2485 00000100:00100000:3.0:1767320154.844955:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985353472:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320154.845006:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320154.971591:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320155.995352:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320156.329379:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320156.329447:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320156.329525:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920601216 00000100:00100000:0.0:1767320156.329543:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920601216 00000100:00100000:3.0:1767320156.329568:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920601088 00000100:00100000:3.0:1767320156.329579:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920601088 00000100:00080000:2.0:1767320156.329670:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320156.329678:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5051 00000100:00100000:3.0:1767320156.329695:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920601216:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320156.329771:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920601216:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (403us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320156.329779:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5051 00000100:00100000:3.0:1767320156.329799:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5673 00000100:00100000:3.0:1767320156.329804:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920601088:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320156.329825:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320156.329848:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920601088:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (402us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320156.329859:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5673 00000100:00080000:2.0:1767320156.379470:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320156.379480:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320156.379483:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320156.379487:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320156.379492:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320156.379509:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5399 00000100:00080000:2.0:1767320156.379513:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320156.379518:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320156.379558:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21180 x1853163985353600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320156.379626:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985353600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.379691:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320156.379796:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353600 00000100:00100000:3.0:1767320156.379804:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353600 00000100:00080000:2.0:1767320156.379841:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767320156.379845:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5674 00000100:00080000:2.0:1767320156.379846:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320156.379848:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320156.379851:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5399 00000100:00100000:3.0:1767320156.379853:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985353600:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767320156.379862:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320156.379866:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320156.379880:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21500 x1853163985353728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00020000:3.0:1767320156.379882:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320156.380080:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985353728:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.380106:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320156.380208:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320156.380211:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320156.380213:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320156.380216:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5399 00000100:00080000:2.0:1767320156.380219:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320156.380222:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320156.380234:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22680 x1853163985353984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320156.380249:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320156.380252:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320156.380253:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320156.380256:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5399 00000100:00080000:2.0:1767320156.380258:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320156.380260:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320156.380268:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22d80 x1853163985354240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320156.380305:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353728 00000100:00100000:0.0:1767320156.380319:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353728 00000100:00100000:2.0:1767320156.380319:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985354496:0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.380373:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320156.380382:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1909 00000100:00100000:0.0:1767320156.380393:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985353728:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320156.380403:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320156.380413:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985354368:0@lo:400:kworker.0 00010000:00020000:0.0:1767320156.380419:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320156.380477:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320156.380575:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985354496 00000100:00100000:2.0:1767320156.380581:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985354496 00000100:00100000:0.0:1767320156.380600:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985353728:12345-0@lo:38:kworker.0 Request processed in 205us (488us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320156.380612:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1909 00000100:00100000:2.0:1767320156.380654:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985354368 00000100:00100000:2.0:1767320156.380660:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985354368 00000100:00100000:2.0:1767320156.380702:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3297 00000100:00100000:2.0:1767320156.380721:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985354496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.380770:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985354496:12345-0@lo:400:kworker.0 Request processed in 60us (410us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320156.381313:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3298 00000100:00100000:2.0:1767320156.381331:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3297 00000100:00100000:0.0:1767320156.381334:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985354368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320156.381393:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985354368:12345-0@lo:400:kworker.0 Request processed in 69us (917us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320156.381401:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3298 00000100:00080000:0.0:1767320156.381441:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320156.381449:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320156.381479:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985353856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.381497:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985354368:0@lo:400:kworker.0 00000100:00100000:0.0:1767320156.381873:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320156.381915:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320156.381922:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985354112:0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.381946:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985354496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320156.381956:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320156.382007:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353856 00000100:00100000:0.0:1767320156.382015:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353856 00000100:00100000:0.0:1767320156.382063:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985354112 00000100:00100000:0.0:1767320156.382066:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985354112 00000100:00100000:0.0:1767320156.382090:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5052 00000100:00100000:0.0:1767320156.382098:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985353856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.382193:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5053 00000100:00100000:2.0:1767320156.382209:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985354112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.382265:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985354112:12345-0@lo:400:kworker.0 Request processed in 63us (316us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320156.382272:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5053 00000100:00080000:2.0:1767320156.383549:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320156.383608:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985353856:12345-0@lo:400:kworker.0 Request processed in 62us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320156.383620:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5052 00000100:00100000:0.0:1767320156.383693:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985354112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320156.383706:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985353856:0@lo:400:kworker.0 00000100:00100000:3.0:1767320156.387540:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985353600:12345-0@lo:38:kworker.0 Request processed in 7677us (7839us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320156.387567:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5674 00000100:00080000:1.0:1767320156.389739:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.389751:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.389756:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985353600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.389791:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985353984:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.390240:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320156.390289:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985354240:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.390314:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320156.390346:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.390350:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.390354:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985353728:0@lo:38:kworker.0 00000100:00100000:0.0:1767320156.390679:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985353984 00000100:00100000:0.0:1767320156.390695:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985353984 00000100:00100000:3.0:1767320156.390718:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985354240 00000100:00100000:3.0:1767320156.390736:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985354240 00000100:00100000:0.0:1767320156.390851:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5675 00000100:00100000:0.0:1767320156.390863:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985353984:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320156.390890:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5676 00010000:00020000:0.0:1767320156.390893:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320156.390903:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985354240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320156.390927:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320156.391003:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985353984:12345-0@lo:38:kworker.0 Request processed in 138us (1146us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320156.391016:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5675 00000100:00100000:3.0:1767320156.391743:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985354240:12345-0@lo:38:kworker.0 Request processed in 838us (1424us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320156.391753:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5676 00000100:00080000:1.0:1767320156.391834:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.391838:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.391842:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985353984:0@lo:38:kworker.0 00000100:00080000:1.0:1767320156.392000:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.392004:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.392007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985354240:0@lo:38:kworker.0 00010000:00010000:2.0:1767320157.019622:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320158.046016:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320159.069441:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320159.964521:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985354624:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320159.964607:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320159.964704:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985354624 00000100:00100000:1.0:1767320159.964716:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985354624 00000100:00100000:1.0:1767320159.964795:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2486 00000100:00100000:1.0:1767320159.964807:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985354624:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320159.964823:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767320159.964847:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320159.964851:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985354752:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320159.964886:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320159.964910:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985354624:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (300us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320159.964921:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2486 00000100:00100000:1.0:1767320159.964942:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985354752 00000100:00100000:1.0:1767320159.964945:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985354752 00000100:00100000:1.0:1767320159.964983:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2487 00000100:00100000:1.0:1767320159.964989:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985354752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320159.965006:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320159.965040:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985354752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 51us (153us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320159.965047:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2487 00000100:00100000:2.0:1767320159.965072:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985354624:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320159.965123:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320159.965156:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985354752:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320160.092887:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320161.116293:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320161.452609:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320161.452788:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320161.453161:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920601472 00000100:00100000:2.0:1767320161.455133:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920601472 00000100:00100000:2.0:1767320161.455256:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920601344 00000100:00100000:2.0:1767320161.455260:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920601344 00000100:00100000:2.0:1767320161.455377:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5054 00000100:00100000:2.0:1767320161.455395:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920601472:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320161.455512:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920601472:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 124us (2917us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320161.455520:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5054 00000100:00100000:2.0:1767320161.455546:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5677 00000100:00100000:2.0:1767320161.455551:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920601344:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320161.455577:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320161.455606:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920601344:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (2818us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320161.455613:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5677 00000100:00080000:0.0:1767320161.457471:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320161.499758:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320161.499769:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320161.499773:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320161.499777:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320161.499783:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320161.499793:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5404 00000100:00080000:3.0:1767320161.499799:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320161.499812:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320161.499860:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985354880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320161.499881:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320161.499885:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320161.499888:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320161.499891:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5404 00000100:00080000:3.0:1767320161.499894:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320161.499899:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320161.499914:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a680 x1853163985355008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320161.499948:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320161.499952:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320161.499954:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320161.499959:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5404 00000100:00080000:3.0:1767320161.499962:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320161.499967:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320161.499983:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518e00 x1853163985355264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320161.500002:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320161.500005:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320161.500007:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320161.500011:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5404 00000100:00080000:3.0:1767320161.500013:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320161.500017:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320161.500027:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518700 x1853163985355520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320161.500055:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985354880:0@lo:38:kworker.0 00000100:00100000:2.0:1767320161.500150:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.500249:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985355008:0@lo:38:kworker.0 00000100:00100000:3.0:1767320161.500281:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985355776:0@lo:400:kworker.0 00000100:00100000:2.0:1767320161.500284:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.500320:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985355264:0@lo:38:kworker.0 00000100:00100000:3.0:1767320161.500330:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.500368:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320161.500376:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320161.500384:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985355648:0@lo:400:kworker.0 00000100:00100000:2.0:1767320161.500394:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985355520:0@lo:38:kworker.0 00000100:00100000:3.0:1767320161.500416:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.500430:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320161.500456:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985354880 00000100:00100000:3.0:1767320161.500465:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985354880 00000100:00100000:2.0:1767320161.500481:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355008 00000100:00100000:2.0:1767320161.500487:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355008 00000100:00100000:3.0:1767320161.500505:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355264 00000100:00100000:3.0:1767320161.500509:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355264 00000100:00100000:3.0:1767320161.500529:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355520 00000100:00100000:3.0:1767320161.500532:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355520 00000100:00100000:3.0:1767320161.500551:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5678 00000100:00100000:3.0:1767320161.500559:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985354880:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320161.500584:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320161.500598:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1910 00000100:00100000:2.0:1767320161.500605:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985355008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320161.500617:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320161.500790:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5679 00000100:00100000:1.0:1767320161.500797:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985355264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320161.500854:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320161.502191:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985354880:12345-0@lo:38:kworker.0 Request processed in 1605us (2015us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320161.502206:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5678 00000100:00100000:3.0:1767320161.502248:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5680 00000100:00100000:3.0:1767320161.502255:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985355520:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320161.502288:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320161.503555:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355776 00000100:00100000:0.0:1767320161.503573:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355776 00000100:00100000:0.0:1767320161.503665:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355648 00000100:00100000:0.0:1767320161.503669:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355648 00000100:00100000:0.0:1767320161.503701:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3299 00000100:00100000:0.0:1767320161.503712:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985355776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.503815:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985355776:12345-0@lo:400:kworker.0 Request processed in 104us (3485us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320161.503823:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3299 00000100:00100000:0.0:1767320161.503842:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3300 00000100:00100000:0.0:1767320161.503848:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985355648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.503884:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985355648:12345-0@lo:400:kworker.0 Request processed in 37us (3469us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320161.503890:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3300 00000100:00080000:0.0:1767320161.503931:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320161.503936:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320161.503989:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985355648:0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.504007:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985355776:0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.504060:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985355136:0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.504105:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320161.504128:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320161.504136:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985355392:0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.504157:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320161.504223:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355136 00000100:00100000:0.0:1767320161.504228:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355136 00000100:00100000:0.0:1767320161.504258:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355392 00000100:00100000:0.0:1767320161.504261:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355392 00000100:00100000:0.0:1767320161.504275:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5055 00000100:00100000:0.0:1767320161.504280:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985355136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.504315:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985355136:12345-0@lo:400:kworker.0 Request processed in 35us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320161.504321:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5055 00000100:00100000:0.0:1767320161.504334:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5056 00000100:00100000:0.0:1767320161.504350:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985355392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.504376:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985355392:12345-0@lo:400:kworker.0 Request processed in 28us (218us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320161.504382:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5056 00000100:00080000:0.0:1767320161.504397:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320161.504413:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985355392:0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.504421:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985355136:0@lo:400:kworker.0 00000100:00080000:0.0:1767320161.504736:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320161.504744:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320161.504749:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985354880:0@lo:38:kworker.0 00000100:00100000:1.0:1767320161.505280:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985355264:12345-0@lo:38:kworker.0 Request processed in 4477us (4909us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320161.505291:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5679 00000100:00100000:3.0:1767320161.505701:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985355520:12345-0@lo:38:kworker.0 Request processed in 3442us (5269us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320161.505711:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5680 00000100:00100000:2.0:1767320161.505787:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985355008:12345-0@lo:38:kworker.0 Request processed in 5179us (5501us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320161.505798:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1910 00000100:00080000:0.0:1767320161.506158:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320161.506479:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320161.506484:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985355008:0@lo:38:kworker.0 00000100:00080000:0.0:1767320161.506505:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320161.506510:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320161.506513:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985355264:0@lo:38:kworker.0 00000100:00080000:0.0:1767320161.506526:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320161.506529:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320161.506532:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985355520:0@lo:38:kworker.0 00010000:00010000:1.0:1767320162.139505:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320163.163468:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320164.191466:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320165.083843:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320165.083866:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985355904:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320165.084043:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320165.084196:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985355904 00000100:00100000:0.0:1767320165.084209:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985355904 00000100:00100000:0.0:1767320165.084307:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2488 00000100:00100000:0.0:1767320165.084318:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985355904:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320165.084360:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320165.084558:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985356032:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320165.084708:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985355904:12345-0@lo:13:osp-pre-0-1.0 Request processed in 391us (665us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320165.084717:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2488 00000100:00100000:1.0:1767320165.084792:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320165.084861:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985355904:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320165.085075:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356032 00000100:00100000:0.0:1767320165.085080:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356032 00000100:00100000:0.0:1767320165.085118:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2489 00000100:00100000:0.0:1767320165.085126:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985356032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320165.085149:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320165.085224:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985356032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (431us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320165.085236:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2489 00000100:00080000:3.0:1767320165.085308:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320165.085313:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320165.085388:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985356032:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320165.214442:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767320165.548126:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920601600 offset 0 length 520: 4 00000100:00100000:2.0:1767320165.548234:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320165.548294:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320165.548361:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920601856 00000100:00100000:3.0:1767320165.548387:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920601856 00000100:00080000:0.0:1767320165.548533:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320165.548580:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920601728 00000100:00100000:3.0:1767320165.548584:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920601728 00000100:00100000:1.0:1767320165.552596:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5057 00000100:00100000:1.0:1767320165.552633:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920601856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320165.552793:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920601856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 181us (4557us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320165.552804:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5057 00000100:00100000:1.0:1767320165.552843:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5681 00000100:00100000:1.0:1767320165.552849:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920601728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320165.552876:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320165.553882:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920601728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1024us (5579us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320165.554054:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5681 00010000:00010000:0.0:1767320166.235531:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320166.619417:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320166.619427:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320166.619430:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320166.619434:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320166.619439:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320166.619447:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5409 00000100:00080000:1.0:1767320166.619452:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320166.619459:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320166.619472:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320141/real 1767320141] req@ffff8ce1a54b7b80 x1853163985349632/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320166 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320166.619499:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9880 x1853163985356160/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320166.619507:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a54b7b80 x1853163985349632/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320166 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320166.619519:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320166.619523:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320166.619524:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320166.619528:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5409 00000100:00080000:1.0:1767320166.619531:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320166.619534:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320166.619539:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985356160:0@lo:38:kworker.0 00000100:00100000:1.0:1767320166.619550:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d500 x1853163985356288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320166.619607:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767320166.619614:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320166.619615:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320166.619619:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320166.619624:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5409 00000100:00080000:1.0:1767320166.619627:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320166.619631:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320166.619645:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7d500 x1853163985356544/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320166.619664:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320166.619667:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320166.619669:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320166.619673:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5409 00000100:00080000:1.0:1767320166.619675:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320166.619677:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320166.619681:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320166.619693:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7e300 x1853163985356800/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320166.619696:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320166.619705:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985349632:0@lo:250:kworker.0 00000100:00100000:0.0:1767320166.619732:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985356288:0@lo:38:kworker.0 00000100:00080000:1.0:1767320166.619740:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320166.619744:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320166.619747:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320166.619752:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5389 00000100:00080000:1.0:1767320166.619755:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320166.619759:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320166.619770:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320166.619775:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7c700 x1853163985357184/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320166.619820:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985356544:0@lo:38:kworker.0 00000100:00100000:0.0:1767320166.619846:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320166.619866:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985356416:0@lo:400:kworker.0 00000100:00100000:0.0:1767320166.619873:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985356800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320166.619879:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985356928:0@lo:400:kworker.0 00000100:00100000:0.0:1767320166.619918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320166.619955:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985357184:0@lo:250:kworker.0 00000100:00100000:2.0:1767320166.619965:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320166.619972:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985356672:0@lo:400:kworker.0 00000400:00000100:0.0:1767320166.619985:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985357184 offset 0 length 520: 4 00000100:00100000:2.0:1767320166.620038:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320166.620038:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320166.620056:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985357056:0@lo:400:kworker.0 00000100:00100000:2.0:1767320166.620107:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320166.620158:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356288 00000100:00100000:2.0:1767320166.620186:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356288 00000100:00100000:2.0:1767320166.620231:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1911 00000100:00100000:2.0:1767320166.620240:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985356288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320166.620281:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320166.620347:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985356288:12345-0@lo:38:kworker.0 Request processed in 106us (578us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320166.620356:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1911 00000100:00100000:1.0:1767320166.626113:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356160 00000100:00100000:0.0:1767320166.626117:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356544 00000100:00100000:0.0:1767320166.626128:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356544 00000100:00100000:1.0:1767320166.626129:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356160 00000100:00100000:0.0:1767320166.626154:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356928 00000100:00100000:0.0:1767320166.626159:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356928 00000100:00100000:1.0:1767320166.626197:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356800 00000100:00100000:1.0:1767320166.626201:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356800 00000100:00100000:0.0:1767320166.626221:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985357056 00000100:00100000:1.0:1767320166.626225:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356672 00000100:00100000:0.0:1767320166.626227:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985357056 00000100:00100000:1.0:1767320166.626231:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356672 00000100:00100000:0.0:1767320166.626269:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3301 00000100:00100000:1.0:1767320166.626279:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5058 00000100:00100000:0.0:1767320166.626282:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985356928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320166.626291:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985356672:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320166.626377:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985356672:12345-0@lo:400:kworker.0 Request processed in 90us (6345us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320166.626385:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985356928:12345-0@lo:400:kworker.0 Request processed in 105us (6421us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320166.626388:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5058 00000100:00100000:1.0:1767320166.626406:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5682 00000100:00100000:0.0:1767320166.626410:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3301 00000100:00100000:1.0:1767320166.626415:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985356160:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320166.626430:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3302 00010000:00020000:1.0:1767320166.626436:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320166.626437:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985357056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320166.626481:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985356160:12345-0@lo:38:kworker.0 Request processed in 67us (6869us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320166.626482:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985357056:12345-0@lo:400:kworker.0 Request processed in 46us (6377us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320166.626495:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3302 00000100:00100000:1.0:1767320166.626500:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5682 00000100:00100000:1.0:1767320166.626519:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5683 00000100:00100000:1.0:1767320166.626524:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985356800:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320166.626534:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320166.626539:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320166.626588:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320166.626592:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320166.626595:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985356288:0@lo:38:kworker.0 00000100:00080000:0.0:1767320166.626602:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320166.626604:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320166.626606:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985356800:0@lo:38:kworker.0 00000100:00080000:0.0:1767320166.626611:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320166.626613:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320166.626615:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985356160:0@lo:38:kworker.0 00000100:00100000:0.0:1767320166.626647:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985356416 00000100:00100000:0.0:1767320166.626651:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985356416 00000100:00100000:0.0:1767320166.626667:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5059 00000100:00100000:0.0:1767320166.626673:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985356416:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320166.626707:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985356416:12345-0@lo:400:kworker.0 Request processed in 34us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320166.626714:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5059 00000100:00100000:0.0:1767320166.626737:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985356416:0@lo:400:kworker.0 00000100:00100000:1.0:1767320166.626757:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985356800:12345-0@lo:38:kworker.0 Request processed in 229us (6838us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320166.626758:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5684 00000100:00100000:0.0:1767320166.626766:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985356544:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320166.626771:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5683 00010000:00020000:0.0:1767320166.626781:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320166.626809:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320166.626816:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985356544:12345-0@lo:38:kworker.0 Request processed in 47us (6966us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320166.626822:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5684 00000100:00080000:1.0:1767320166.626849:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320166.626855:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320166.626859:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985356544:0@lo:38:kworker.0 00000100:00100000:0.0:1767320166.626888:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985357056:0@lo:400:kworker.0 00000100:00100000:1.0:1767320166.626905:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985356672:0@lo:400:kworker.0 00000100:00100000:0.0:1767320166.626909:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985356928:0@lo:400:kworker.0 00010000:00010000:1.0:1767320167.259372:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320168.287836:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320169.307714:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320170.203733:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985357312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320170.203788:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985357440:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320170.203846:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320170.204040:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320170.204140:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985357440 00000100:00100000:0.0:1767320170.204150:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985357440 00000100:00100000:0.0:1767320170.204269:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985357312 00000100:00100000:0.0:1767320170.204272:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985357312 00000100:00100000:0.0:1767320170.204300:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2490 00000100:00100000:0.0:1767320170.204310:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985357440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320170.204366:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320170.204418:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985357440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (571us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320170.204427:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2490 00000100:00100000:0.0:1767320170.204447:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2491 00000100:00100000:0.0:1767320170.204454:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985357312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320170.204474:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320170.204505:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985357312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (465us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320170.204512:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2491 00000100:00100000:0.0:1767320170.204563:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985357312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320170.204563:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5ba5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985357440:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320170.204790:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320170.204795:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320170.331842:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320170.666132:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320170.666255:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320170.666336:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920601984 00000100:00100000:1.0:1767320170.666349:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920601984 00000100:00100000:1.0:1767320170.666406:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920602112 00000100:00100000:1.0:1767320170.666410:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920602112 00000100:00100000:1.0:1767320170.666466:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5060 00000100:00100000:1.0:1767320170.666478:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920602112:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320170.666580:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920602112:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 103us (323us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320170.666590:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5060 00000100:00100000:1.0:1767320170.666613:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5685 00000100:00100000:1.0:1767320170.666618:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920601984:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320170.666643:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320170.666673:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920601984:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (557us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320170.666681:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5685 00000100:00080000:1.0:1767320170.666825:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320171.356748:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320171.741654:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320171.741670:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320171.741674:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320171.741679:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320171.741685:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320171.741697:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5414 00000100:00080000:3.0:1767320171.741703:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320171.741713:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320171.741774:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518700 x1853163985357568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320171.741812:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320171.741816:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320171.741818:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320171.741822:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5414 00000100:00080000:3.0:1767320171.741825:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320171.741829:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320171.741844:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518e00 x1853163985357696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320171.741892:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320171.741897:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320171.741899:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320171.741903:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5414 00000100:00080000:3.0:1767320171.741905:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320171.741909:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320171.741921:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985357952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320171.741939:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320171.741943:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320171.741945:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320171.741949:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5414 00000100:00080000:3.0:1767320171.741951:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320171.741954:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320171.741965:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351ad80 x1853163985358208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320171.742037:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985358464:0@lo:400:kworker.0 00000100:00100000:3.0:1767320171.742131:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320171.742215:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985358336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320171.742257:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320171.742297:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985358080:0@lo:400:kworker.0 00000100:00100000:2.0:1767320171.742313:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358464 00000100:00100000:2.0:1767320171.742324:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358464 00000100:00100000:2.0:1767320171.742377:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358336 00000100:00100000:2.0:1767320171.742381:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358336 00000100:00100000:2.0:1767320171.742402:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3303 00000100:00100000:2.0:1767320171.742412:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985358464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320171.742481:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3304 00000100:00100000:1.0:1767320171.742505:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985358336:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320171.742665:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985357824:0@lo:400:kworker.0 00000100:00100000:2.0:1767320171.743730:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985358464:12345-0@lo:400:kworker.0 Request processed in 1311us (1590us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320171.743745:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3303 00000100:00100000:2.0:1767320171.743831:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985358464:0@lo:400:kworker.0 00000100:00100000:2.0:1767320171.743918:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985357568:0@lo:38:kworker.0 00000100:00100000:2.0:1767320171.743960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320171.744000:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985357696:0@lo:38:kworker.0 00000100:00100000:2.0:1767320171.744026:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320171.744049:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985357952:0@lo:38:kworker.0 00000100:00100000:2.0:1767320171.744068:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320171.744086:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985358208:0@lo:38:kworker.0 00000100:00100000:2.0:1767320171.744109:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320171.746096:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985357568 00000100:00100000:2.0:1767320171.746117:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985357568 00000100:00100000:2.0:1767320171.746219:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985357952 00000100:00100000:2.0:1767320171.746222:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985357952 00000100:00100000:2.0:1767320171.746239:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358208 00000100:00100000:2.0:1767320171.746241:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358208 00000100:00100000:2.0:1767320171.746263:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5686 00000100:00100000:2.0:1767320171.746272:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985357568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320171.746303:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320171.746407:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985357568:12345-0@lo:38:kworker.0 Request processed in 135us (2446us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320171.746414:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5686 00000100:00100000:2.0:1767320171.746433:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5687 00000100:00100000:2.0:1767320171.746437:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985357952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320171.746446:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320171.746466:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985357952:12345-0@lo:38:kworker.0 Request processed in 29us (2398us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320171.746472:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5687 00000100:00100000:2.0:1767320171.746480:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5688 00000100:00100000:2.0:1767320171.746484:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985358208:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320171.746495:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320171.747621:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985358208:12345-0@lo:38:kworker.0 Request processed in 28us (2404us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320171.747640:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5688 00000100:00080000:2.0:1767320171.747733:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320171.747739:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320171.747743:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985357568:0@lo:38:kworker.0 00000100:00080000:2.0:1767320171.747755:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320171.747759:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320171.747762:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985357952:0@lo:38:kworker.0 00000100:00080000:2.0:1767320171.747770:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320171.747773:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320171.747776:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985358208:0@lo:38:kworker.0 00000100:00100000:2.0:1767320171.747880:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985357696 00000100:00100000:2.0:1767320171.747886:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985357696 00000100:00100000:2.0:1767320171.747920:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1912 00000100:00100000:2.0:1767320171.747926:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985357696:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320171.747946:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320171.748008:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985357696:12345-0@lo:38:kworker.0 Request processed in 81us (3980us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320171.748013:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1912 00000100:00080000:2.0:1767320171.748034:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320171.748037:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320171.748040:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985357696:0@lo:38:kworker.0 00000100:00100000:1.0:1767320171.751330:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985358336:12345-0@lo:400:kworker.0 Request processed in 8824us (9057us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320171.751350:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3304 00000100:00100000:2.0:1767320171.752040:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985358336:0@lo:400:kworker.0 00000100:00100000:3.0:1767320171.754665:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320171.754849:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358080 00000100:00100000:2.0:1767320171.754862:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358080 00000100:00100000:2.0:1767320171.754986:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5061 00000100:00100000:2.0:1767320171.755000:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985358080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320171.755073:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985358080:12345-0@lo:400:kworker.0 Request processed in 77us (421us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320171.755093:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5061 00000100:00100000:0.0:1767320171.756218:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320171.756269:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985357824 00000100:00100000:0.0:1767320171.756277:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985357824 00000100:00100000:3.0:1767320171.756726:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985358080:0@lo:400:kworker.0 00000100:00080000:1.0:1767320171.756898:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320171.757053:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5062 00000100:00100000:0.0:1767320171.757064:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985357824:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320171.757142:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985357824:12345-0@lo:400:kworker.0 Request processed in 81us (931us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320171.757159:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5062 00000100:00100000:2.0:1767320171.757352:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985357824:0@lo:400:kworker.0 00010000:00010000:0.0:1767320172.379725:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320173.405423:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320174.428675:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320175.323794:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985358720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320175.323894:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320175.324020:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985358592:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320175.324065:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320175.324111:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358720 00000100:00100000:3.0:1767320175.324121:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358720 00000100:00100000:3.0:1767320175.324277:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358592 00000100:00100000:3.0:1767320175.324281:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358592 00000100:00100000:3.0:1767320175.324306:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2492 00000100:00100000:3.0:1767320175.324315:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985358720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320175.324351:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320175.324453:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2493 00000100:00100000:1.0:1767320175.324462:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985358592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320175.324479:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320175.324513:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985358592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 52us (447us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320175.324521:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2493 00000100:00080000:1.0:1767320175.324548:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320175.324553:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320175.325253:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985358592:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320175.325409:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985358720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1094us (1514us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320175.325418:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2492 00000100:00100000:2.0:1767320175.325492:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985358720:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320175.455441:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320175.790339:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320175.790788:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320175.790871:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920602240 00000100:00100000:0.0:1767320175.790885:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920602240 00000100:00100000:0.0:1767320175.790944:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920602368 00000100:00100000:0.0:1767320175.790948:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920602368 00000100:00100000:0.0:1767320175.791016:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5063 00000100:00080000:1.0:1767320175.791021:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320175.791031:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920602368:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320175.791154:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920602368:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 128us (367us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320175.791214:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5063 00000100:00100000:0.0:1767320175.791259:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5689 00000100:00100000:0.0:1767320175.791265:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920602240:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320175.791291:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320175.791322:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920602240:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (1000us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320175.791330:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5689 00010000:00010000:1.0:1767320176.478398:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320176.859992:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320176.860006:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320176.860010:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320176.860014:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320176.860020:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320176.860031:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5420 00000100:00080000:2.0:1767320176.860037:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320176.860045:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320176.860121:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6a00 x1853163985358848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320176.861376:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767320176.861661:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985358848:0@lo:38:kworker.0 00000100:00100000:0.0:1767320176.861758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320176.861975:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320176.861979:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320176.861984:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5420 00000100:00080000:2.0:1767320176.861987:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320176.861994:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320176.862030:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5f80 x1853163985358976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320176.862090:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320176.862094:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320176.862096:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320176.862100:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5420 00000100:00080000:2.0:1767320176.862103:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320176.862106:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320176.862117:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7800 x1853163985359232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320176.862133:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320176.862136:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320176.862138:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320176.862141:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5420 00000100:00080000:2.0:1767320176.862143:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320176.862147:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320176.862156:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7b80 x1853163985359488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320176.862241:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985359360:0@lo:400:kworker.0 00000100:00100000:2.0:1767320176.862286:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320176.862326:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985358976:0@lo:38:kworker.0 00000100:00100000:0.0:1767320176.862368:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320176.862418:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985359232:0@lo:38:kworker.0 00000100:00100000:0.0:1767320176.862460:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358976 00000100:00100000:0.0:1767320176.862470:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358976 00000100:00100000:0.0:1767320176.862517:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1913 00000100:00100000:0.0:1767320176.862527:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190db9850 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985358976:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320176.862565:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320176.862618:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190db9850 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985358976:12345-0@lo:38:kworker.0 Request processed in 92us (248us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320176.862626:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1913 00000100:00100000:0.0:1767320176.862851:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985359744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320176.862880:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320176.862913:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320176.862919:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985359616:0@lo:400:kworker.0 00000100:00100000:0.0:1767320176.862940:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320176.863090:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985359104:0@lo:400:kworker.0 00000100:00100000:0.0:1767320176.863123:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985359744 00000100:00100000:0.0:1767320176.863129:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985359744 00000100:00100000:0.0:1767320176.863201:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985359616 00000100:00100000:0.0:1767320176.863205:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985359616 00000100:00100000:3.0:1767320176.863214:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3305 00000100:00100000:3.0:1767320176.863227:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985359744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320176.863235:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3306 00000100:00100000:0.0:1767320176.863245:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985359616:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320176.863291:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985359616:12345-0@lo:400:kworker.0 Request processed in 48us (350us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320176.863298:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3306 00000100:00100000:0.0:1767320176.863342:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985359616:0@lo:400:kworker.0 00000100:00100000:3.0:1767320176.863641:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985359744:12345-0@lo:400:kworker.0 Request processed in 418us (759us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320176.863649:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3305 00000100:00100000:0.0:1767320176.864594:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985359744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320176.864861:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985358848 00000100:00100000:1.0:1767320176.864869:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985358848 00000100:00100000:1.0:1767320176.864908:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985359360 00000100:00100000:1.0:1767320176.864911:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985359360 00000100:00100000:1.0:1767320176.864951:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5064 00000100:00100000:1.0:1767320176.864961:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985359360:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320176.865083:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5690 00000100:00100000:3.0:1767320176.865090:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985358848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320176.865115:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320176.865155:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320176.865235:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320176.865248:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985359104 00000100:00100000:0.0:1767320176.865252:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985359104 00000100:00100000:0.0:1767320176.865283:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5065 00000100:00100000:0.0:1767320176.865288:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985359104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320176.865351:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985359104:12345-0@lo:400:kworker.0 Request processed in 63us (196us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320176.865359:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5065 00000100:00080000:0.0:1767320176.865376:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320176.865657:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985359104:0@lo:400:kworker.0 00000100:00100000:0.0:1767320176.865710:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320176.865734:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985359488:0@lo:38:kworker.0 00000100:00100000:0.0:1767320176.865764:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320176.865792:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320176.865797:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320176.865802:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985358976:0@lo:38:kworker.0 00000100:00100000:0.0:1767320176.865821:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985359232 00000100:00100000:0.0:1767320176.865824:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985359232 00000100:00100000:0.0:1767320176.865838:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985359488 00000100:00100000:0.0:1767320176.865840:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985359488 00000100:00100000:0.0:1767320176.865849:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5691 00000100:00100000:0.0:1767320176.865853:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985359232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320176.865866:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320176.865893:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985359232:12345-0@lo:38:kworker.0 Request processed in 39us (182us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320176.865899:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5691 00000100:00100000:0.0:1767320176.865909:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5692 00000100:00100000:0.0:1767320176.865912:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985359488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320176.865920:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320176.865937:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985359488:12345-0@lo:38:kworker.0 Request processed in 24us (173us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320176.865943:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5692 00000100:00080000:0.0:1767320176.865964:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320176.865968:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320176.865972:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985359232:0@lo:38:kworker.0 00000100:00080000:0.0:1767320176.865980:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320176.865983:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320176.865986:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985359488:0@lo:38:kworker.0 00000100:00100000:1.0:1767320176.866118:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985359360:12345-0@lo:400:kworker.0 Request processed in 1159us (3830us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320176.866126:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5064 00000100:00100000:2.0:1767320176.866262:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985359360:0@lo:400:kworker.0 00000100:00100000:3.0:1767320176.866878:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985358848:12345-0@lo:38:kworker.0 Request processed in 1784us (5116us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320176.866887:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5690 00000100:00080000:0.0:1767320176.866945:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320176.866950:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320176.866954:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985358848:0@lo:38:kworker.0 00010000:00010000:0.0:1767320177.499767:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320178.523565:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320179.547479:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320180.443661:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985359872:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320180.443758:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320180.444865:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985359872 00000100:00100000:0.0:1767320180.444881:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985359872 00000100:00100000:0.0:1767320180.445012:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2494 00000100:00100000:0.0:1767320180.445025:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985359872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320180.445068:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320180.445158:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985359872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 136us (1399us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320180.445190:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2494 00000100:00100000:0.0:1767320180.445255:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985359872:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320180.445347:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320180.446103:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985360000:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320180.446158:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320180.446272:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360000 00000100:00100000:1.0:1767320180.446280:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360000 00000100:00100000:1.0:1767320180.446337:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2495 00000100:00100000:1.0:1767320180.446346:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985360000:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320180.446370:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767320180.446439:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320180.446810:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985360000:12345-0@lo:13:osp-pre-0-1.0 Request processed in 277us (462us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320180.446822:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2495 00000100:00100000:3.0:1767320180.447133:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985360000:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320180.571411:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320180.905910:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320180.906065:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920602624 00000100:00100000:3.0:1767320180.906077:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920602624 00000100:00100000:3.0:1767320180.906155:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5066 00000100:00080000:0.0:1767320180.906194:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320180.906219:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920602624:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320180.906313:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920602624:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 149us (416us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320180.906320:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5066 00000100:00100000:1.0:1767320180.909666:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320180.909760:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920602496 00000100:00100000:2.0:1767320180.909772:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920602496 00000100:00100000:2.0:1767320180.909820:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5693 00000100:00100000:2.0:1767320180.909829:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920602496:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320180.909852:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320180.909912:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920602496:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 83us (255us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320180.909920:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5693 00010000:00010000:2.0:1767320181.595730:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320181.980694:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320181.980705:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320181.980708:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320181.980714:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320181.980719:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320181.980728:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5425 00000100:00080000:3.0:1767320181.980732:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320181.980739:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320181.980783:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4a80 x1853163985360128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320181.980822:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320181.980825:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320181.980827:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320181.980831:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5425 00000100:00080000:3.0:1767320181.980833:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320181.980837:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320181.980852:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6d80 x1853163985360256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320181.980901:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320181.980905:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320181.980907:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320181.980911:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5425 00000100:00080000:3.0:1767320181.980913:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320181.980916:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320181.980928:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985360512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320181.980943:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320181.980947:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320181.980949:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320181.980952:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5425 00000100:00080000:3.0:1767320181.980955:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320181.980969:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320181.980980:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0000 x1853163985360768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320181.981042:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985360896:0@lo:400:kworker.0 00000100:00100000:3.0:1767320181.981111:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320181.981213:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320181.981219:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985361024:0@lo:400:kworker.0 00000100:00100000:3.0:1767320181.981241:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320181.981276:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360896 00000100:00100000:3.0:1767320181.981284:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360896 00000100:00100000:3.0:1767320181.981323:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361024 00000100:00100000:3.0:1767320181.981325:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361024 00000100:00100000:3.0:1767320181.981344:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3307 00000100:00100000:3.0:1767320181.981352:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985360896:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320181.981509:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985360640:0@lo:400:kworker.0 00000100:00100000:2.0:1767320181.981563:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985360384:0@lo:400:kworker.0 00000100:00100000:0.0:1767320181.981855:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3308 00000100:00100000:0.0:1767320181.981866:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985361024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320181.991240:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320181.991361:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985360128:0@lo:38:kworker.0 00000100:00100000:1.0:1767320181.991405:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320181.991434:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985360256:0@lo:38:kworker.0 00000100:00100000:1.0:1767320181.991460:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320181.991484:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985360512:0@lo:38:kworker.0 00000100:00100000:1.0:1767320181.991504:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320181.991528:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985360768:0@lo:38:kworker.0 00000100:00100000:1.0:1767320181.991548:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320181.991584:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360256 00000100:00100000:1.0:1767320181.991592:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360256 00000100:00100000:1.0:1767320181.991625:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1914 00000100:00100000:1.0:1767320181.991632:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985360256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320181.991653:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320181.991695:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985360256:12345-0@lo:38:kworker.0 Request processed in 63us (234us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320181.991702:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1914 00000100:00080000:1.0:1767320181.991740:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320181.991746:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320181.991750:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985360256:0@lo:38:kworker.0 00000100:00100000:1.0:1767320181.991783:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360640 00000100:00100000:1.0:1767320181.991787:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360640 00000100:00100000:1.0:1767320181.991839:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360128 00000100:00100000:1.0:1767320181.991842:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360128 00000100:00100000:1.0:1767320181.991855:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360512 00000100:00100000:1.0:1767320181.991857:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360512 00000100:00100000:1.0:1767320181.991863:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360768 00000100:00100000:1.0:1767320181.991865:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360768 00000100:00100000:1.0:1767320181.991873:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5067 00000100:00100000:1.0:1767320181.991879:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985360640:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320181.991924:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985360640:12345-0@lo:400:kworker.0 Request processed in 46us (694us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320181.991930:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5067 00000100:00100000:1.0:1767320181.991942:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5694 00000100:00100000:1.0:1767320181.991946:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985360128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320181.991955:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320181.993062:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985360128:12345-0@lo:38:kworker.0 Request processed in 1111us (1653us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320181.993071:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5694 00000100:00100000:1.0:1767320181.993092:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5695 00000100:00100000:1.0:1767320181.993096:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985360512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320181.993111:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320181.993134:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985360512:12345-0@lo:38:kworker.0 Request processed in 37us (1629us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320181.993140:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5695 00000100:00100000:1.0:1767320181.993149:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5696 00000100:00100000:1.0:1767320181.993153:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985360768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320181.993177:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320181.993201:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985360768:12345-0@lo:38:kworker.0 Request processed in 47us (1652us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320181.993207:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5696 00000100:00100000:1.0:1767320181.993250:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985360640:0@lo:400:kworker.0 00000100:00080000:1.0:1767320181.993280:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320181.993285:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320181.993288:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985360128:0@lo:38:kworker.0 00000100:00080000:1.0:1767320181.993296:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320181.993298:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320181.993301:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985360512:0@lo:38:kworker.0 00000100:00080000:1.0:1767320181.993307:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320181.993310:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320181.993312:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985360768:0@lo:38:kworker.0 00000100:00080000:1.0:1767320181.993348:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320181.993765:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320181.993968:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985360384 00000100:00100000:1.0:1767320181.993974:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985360384 00000100:00100000:1.0:1767320181.994029:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5068 00000100:00100000:1.0:1767320181.994036:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985360384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320181.994079:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985360384:12345-0@lo:400:kworker.0 Request processed in 43us (313us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320181.994085:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5068 00000100:00080000:2.0:1767320181.994143:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320181.994178:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985360384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320181.997339:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985360896:12345-0@lo:400:kworker.0 Request processed in 15978us (16218us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320181.997355:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3307 00000100:00100000:0.0:1767320181.997589:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985361024:12345-0@lo:400:kworker.0 Request processed in 15724us (16344us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320181.997596:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3308 00000100:00100000:2.0:1767320181.997728:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985360896:0@lo:400:kworker.0 00000100:00100000:2.0:1767320181.997757:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985361024:0@lo:400:kworker.0 00010000:00010000:1.0:1767320182.619722:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320183.643622:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320184.667352:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320185.565775:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320185.570243:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985361280:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320185.570314:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320185.570601:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985361152:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320185.570626:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320185.585141:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361280 00000100:00100000:0.0:1767320185.585156:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361280 00000100:00100000:0.0:1767320185.585252:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361152 00000100:00100000:0.0:1767320185.585256:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361152 00000100:00100000:0.0:1767320185.585280:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2496 00000100:00100000:0.0:1767320185.585288:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985361280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320185.585333:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320185.585409:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985361280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 121us (15095us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320185.585417:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2496 00000100:00100000:0.0:1767320185.585434:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2497 00000100:00100000:0.0:1767320185.585441:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985361152:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320185.585456:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320185.585483:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985361152:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (14856us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320185.585489:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2497 00000100:00080000:1.0:1767320185.586057:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320185.586062:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320185.586116:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985361280:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320185.586130:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985361152:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320185.691482:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320186.026829:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320186.027694:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320186.033552:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920602880 00000100:00100000:1.0:1767320186.033568:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920602880 00000100:00100000:1.0:1767320186.033749:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920602752 00000100:00100000:1.0:1767320186.033754:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920602752 00000100:00100000:1.0:1767320186.033785:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5069 00000100:00100000:1.0:1767320186.033795:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920602880:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320186.034067:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920602880:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 88us (7060us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320186.034077:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5069 00000100:00100000:1.0:1767320186.034098:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5697 00000100:00100000:1.0:1767320186.034103:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920602752:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320186.034124:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320186.034154:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920602752:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (6462us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320186.034159:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5697 00000100:00080000:3.0:1767320186.034777:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320186.715468:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320187.099329:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320187.099339:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320187.099343:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320187.099346:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320187.099350:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320187.099358:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5430 00000100:00080000:2.0:1767320187.099362:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320187.099368:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320187.099418:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5f80 x1853163985361408/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320187.099445:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320187.099448:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320187.099450:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320187.099453:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5430 00000100:00080000:2.0:1767320187.099455:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320187.099458:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320187.099471:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4e00 x1853163985361536/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320187.099504:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320187.099509:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320187.099511:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320187.099515:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5430 00000100:00080000:2.0:1767320187.099517:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320187.099520:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320187.099531:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5c00 x1853163985361792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320187.099550:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320187.099553:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320187.099555:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320187.099558:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5430 00000100:00080000:2.0:1767320187.099560:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320187.099564:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320187.099575:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7800 x1853163985362048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320187.099627:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985362176:0@lo:400:kworker.0 00000100:00100000:2.0:1767320187.099690:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320187.099749:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320187.099756:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985362304:0@lo:400:kworker.0 00000100:00100000:2.0:1767320187.099779:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320187.100040:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985362176 00000100:00100000:0.0:1767320187.100057:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985362176 00000100:00100000:0.0:1767320187.100117:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3309 00000100:00100000:0.0:1767320187.100127:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985362176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320187.100213:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985362176:12345-0@lo:400:kworker.0 Request processed in 88us (522us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320187.100220:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3309 00000100:00100000:0.0:1767320187.100236:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985362304 00000100:00100000:0.0:1767320187.100239:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985362304 00000100:00100000:0.0:1767320187.100259:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3310 00000100:00100000:0.0:1767320187.100264:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985362304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320187.100292:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985362304:12345-0@lo:400:kworker.0 Request processed in 27us (512us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320187.100297:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3310 00000100:00100000:0.0:1767320187.100317:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985361664:0@lo:400:kworker.0 00000100:00100000:0.0:1767320187.100344:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320187.100368:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320187.100373:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985361920:0@lo:400:kworker.0 00000100:00100000:3.0:1767320187.100402:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985361408:0@lo:38:kworker.0 00000100:00100000:0.0:1767320187.100406:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320187.100429:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320187.100431:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361664 00000100:00100000:0.0:1767320187.100434:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361664 00000100:00100000:3.0:1767320187.100444:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985361536:0@lo:38:kworker.0 00000100:00100000:0.0:1767320187.100457:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361920 00000100:00100000:0.0:1767320187.100460:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361920 00000100:00100000:3.0:1767320187.100464:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320187.100475:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361408 00000100:00100000:0.0:1767320187.100477:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361408 00000100:00100000:3.0:1767320187.100482:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985361792:0@lo:38:kworker.0 00000100:00100000:0.0:1767320187.100489:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5070 00000100:00100000:0.0:1767320187.100494:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985361664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320187.100499:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320187.100521:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985362048:0@lo:38:kworker.0 00000100:00100000:0.0:1767320187.100526:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985361664:12345-0@lo:400:kworker.0 Request processed in 32us (182us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320187.100532:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5070 00000100:00100000:3.0:1767320187.100539:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320187.100541:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361792 00000100:00100000:0.0:1767320187.100545:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361792 00000100:00100000:0.0:1767320187.100552:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985362048 00000100:00100000:0.0:1767320187.100554:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985362048 00000100:00100000:0.0:1767320187.100563:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5071 00000100:00100000:0.0:1767320187.100568:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985361920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320187.100586:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985362304:0@lo:400:kworker.0 00000100:00100000:0.0:1767320187.100593:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985361920:12345-0@lo:400:kworker.0 Request processed in 26us (187us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320187.100597:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985362176:0@lo:400:kworker.0 00000100:00100000:0.0:1767320187.100599:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5071 00000100:00100000:0.0:1767320187.100609:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5698 00000100:00100000:0.0:1767320187.100612:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985361408:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320187.100630:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320187.100670:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985361408:12345-0@lo:38:kworker.0 Request processed in 55us (240us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320187.100676:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5698 00000100:00100000:0.0:1767320187.100687:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5699 00000100:00100000:0.0:1767320187.100690:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985361792:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320187.100699:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320187.100720:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985361792:12345-0@lo:38:kworker.0 Request processed in 28us (219us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320187.100725:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5699 00000100:00100000:0.0:1767320187.100736:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5700 00000100:00100000:0.0:1767320187.100740:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985362048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320187.100748:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320187.100769:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985362048:12345-0@lo:38:kworker.0 Request processed in 28us (230us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320187.100774:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5700 00000100:00100000:0.0:1767320187.100797:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985361920:0@lo:400:kworker.0 00000100:00100000:0.0:1767320187.100806:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985361664:0@lo:400:kworker.0 00000100:00080000:3.0:1767320187.100970:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320187.100976:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320187.100979:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985361408:0@lo:38:kworker.0 00000100:00080000:3.0:1767320187.100987:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320187.100990:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320187.100993:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985361792:0@lo:38:kworker.0 00000100:00080000:3.0:1767320187.101000:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320187.101003:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320187.101005:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985362048:0@lo:38:kworker.0 00000100:00080000:2.0:1767320187.101007:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320187.101011:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320187.101012:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320187.101114:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985361536 00000100:00100000:1.0:1767320187.101119:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985361536 00000100:00100000:1.0:1767320187.101148:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1915 00000100:00100000:1.0:1767320187.101153:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985361536:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320187.101195:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320187.101236:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985361536:12345-0@lo:38:kworker.0 Request processed in 82us (770us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320187.101243:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1915 00000100:00080000:2.0:1767320187.101354:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320187.101358:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320187.101362:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985361536:0@lo:38:kworker.0 00010000:00010000:2.0:1767320187.739413:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320188.763993:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320189.787610:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320190.683658:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985362432:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320190.683716:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320190.683844:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985362432 00000100:00100000:2.0:1767320190.683852:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985362432 00000100:00100000:2.0:1767320190.683920:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2498 00000100:00100000:2.0:1767320190.683931:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985362432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320190.683961:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320190.684008:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985362432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (290us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320190.684015:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2498 00000100:00100000:2.0:1767320190.684061:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985362432:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320190.684128:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320190.684665:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985362560:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320190.684700:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320190.685186:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985362560 00000100:00100000:3.0:1767320190.685191:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985362560 00000100:00100000:3.0:1767320190.685247:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2499 00000100:00100000:3.0:1767320190.685255:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985362560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320190.685274:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320190.685313:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985362560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (612us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320190.685320:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2499 00000100:00080000:0.0:1767320190.685371:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320190.685416:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985362560:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320190.811552:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320191.146353:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767320191.147106:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920603008 offset 0 length 520: 4 00000100:00100000:0.0:1767320191.147138:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320191.152941:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920603264 00000100:00100000:3.0:1767320191.152963:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920603264 00000100:00100000:2.0:1767320191.153085:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920603136 00000100:00100000:2.0:1767320191.153092:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920603136 00000100:00100000:3.0:1767320191.153126:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5072 00000100:00100000:2.0:1767320191.153136:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5701 00000100:00100000:3.0:1767320191.153143:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920603264:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320191.153148:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920603136:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320191.153196:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320191.153475:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320191.153531:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920603136:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 383us (6390us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320191.153534:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920603264:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 395us (7191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320191.153545:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5701 00000100:00100000:3.0:1767320191.153563:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5072 00000100:00100000:2.0:1767320191.195344:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320166/real 1767320166] req@ffff8ce19db7c700 x1853163985357184/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320191 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320191.195389:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19db7c700 x1853163985357184/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320191 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320191.195423:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320191.195455:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320191.195460:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985357184:0@lo:250:kworker.0 00000100:00080000:3.0:1767320191.195806:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320191.195815:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320191.195819:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320191.195826:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5414 00000100:00080000:3.0:1767320191.195831:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320191.195842:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320191.195896:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6a00 x1853163985362688/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320191.195929:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320191.195934:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320191.195936:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320191.195940:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5435 00000100:00080000:3.0:1767320191.195943:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320191.195946:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320191.195964:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5c00 x1853163985362816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320191.195974:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320191.195977:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320191.195978:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320191.195983:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5435 00000100:00080000:3.0:1767320191.195985:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320191.195989:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320191.196001:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7480 x1853163985362944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320191.196057:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320191.196062:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320191.196064:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320191.196068:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5435 00000100:00080000:3.0:1767320191.196071:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320191.196324:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320191.196341:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7100 x1853163985363200/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320191.196362:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320191.196366:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320191.196368:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320191.196372:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5435 00000100:00080000:3.0:1767320191.196375:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320191.196455:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320191.196473:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4e00 x1853163985363456/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320191.196480:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985363328:0@lo:400:kworker.0 00000100:00100000:3.0:1767320191.196567:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985363712:0@lo:400:kworker.0 00000100:00100000:1.0:1767320191.196593:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320191.196617:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320191.196685:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320191.196694:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985363072:0@lo:400:kworker.0 00000100:00100000:1.0:1767320191.196724:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320191.196729:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363712 00000100:00100000:0.0:1767320191.196743:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363712 00000100:00100000:0.0:1767320191.196828:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3311 00000100:00100000:0.0:1767320191.196840:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985363712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320191.196908:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985363712:12345-0@lo:400:kworker.0 Request processed in 69us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320191.196916:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3311 00000100:00100000:0.0:1767320191.196956:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363328 00000100:00100000:0.0:1767320191.196960:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363328 00000100:00100000:0.0:1767320191.196995:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363072 00000100:00100000:0.0:1767320191.196999:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363072 00000100:00100000:0.0:1767320191.197018:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5073 00000100:00100000:0.0:1767320191.197024:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985363328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320191.197064:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985363328:12345-0@lo:400:kworker.0 Request processed in 40us (473us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320191.197071:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5073 00000100:00100000:3.0:1767320191.198593:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985363712:0@lo:400:kworker.0 00000100:00100000:3.0:1767320191.198624:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320191.198636:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985363584:0@lo:400:kworker.0 00000100:00100000:3.0:1767320191.198702:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320191.198765:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363584 00000100:00100000:3.0:1767320191.198771:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363584 00000100:00100000:3.0:1767320191.198827:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3312 00000100:00100000:3.0:1767320191.198835:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985363584:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320191.198887:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985363584:12345-0@lo:400:kworker.0 Request processed in 53us (184us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320191.198895:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3312 00000100:00100000:3.0:1767320191.198930:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985363584:0@lo:400:kworker.0 00000100:00100000:3.0:1767320191.198966:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985362688:0@lo:250:kworker.0 00000400:00000100:3.0:1767320191.198989:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985362688 offset 0 length 520: 4 00000100:00100000:3.0:1767320191.199014:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985362816:0@lo:38:kworker.0 00000100:00100000:3.0:1767320191.199040:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320191.199059:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985362944:0@lo:38:kworker.0 00000100:00100000:3.0:1767320191.199100:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320191.199116:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985363200:0@lo:38:kworker.0 00000100:00100000:3.0:1767320191.199138:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320191.199155:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985363456:0@lo:38:kworker.0 00000100:00100000:3.0:1767320191.199204:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320191.199229:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985362816 00000100:00100000:3.0:1767320191.199233:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985362816 00000100:00100000:0.0:1767320191.199279:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5074 00000100:00100000:0.0:1767320191.199293:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985363072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320191.199335:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985363072:12345-0@lo:400:kworker.0 Request processed in 47us (2612us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320191.199416:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985362944 00000100:00100000:1.0:1767320191.199421:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985362944 00000100:00100000:1.0:1767320191.199450:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1916 00000100:00100000:1.0:1767320191.199455:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985362944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320191.199480:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320191.199523:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985362944:12345-0@lo:38:kworker.0 Request processed in 67us (422us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320191.199530:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1916 00000100:00100000:1.0:1767320191.199557:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363200 00000100:00100000:1.0:1767320191.199561:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363200 00000100:00100000:2.0:1767320191.210525:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363456 00000100:00100000:2.0:1767320191.210544:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363456 00000100:00100000:2.0:1767320191.210586:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5702 00000100:00100000:2.0:1767320191.210597:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985363456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320191.210626:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320191.210723:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985363456:12345-0@lo:38:kworker.0 Request processed in 127us (11518us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320191.210732:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5702 00000100:00100000:2.0:1767320191.210863:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985363072:0@lo:400:kworker.0 00000100:00100000:2.0:1767320191.210878:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985363328:0@lo:400:kworker.0 00000100:00080000:2.0:1767320191.210924:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320191.210930:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320191.210933:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985362944:0@lo:38:kworker.0 00000100:00080000:2.0:1767320191.210944:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320191.210947:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320191.210950:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985363456:0@lo:38:kworker.0 00000100:00080000:2.0:1767320191.210986:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320191.212529:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5703 00000100:00100000:3.0:1767320191.212543:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985362816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320191.212576:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320191.212680:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985362816:12345-0@lo:38:kworker.0 Request processed in 137us (13637us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320191.212688:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5703 00000100:00080000:3.0:1767320191.212777:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320191.212783:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320191.212798:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985362816:0@lo:38:kworker.0 00000100:00100000:0.0:1767320191.222393:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5074 00000100:00100000:1.0:1767320191.222763:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5704 00000100:00100000:1.0:1767320191.222776:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985363200:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320191.222819:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320191.222947:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985363200:12345-0@lo:38:kworker.0 Request processed in 172us (23807us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320191.222955:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5704 00000100:00080000:3.0:1767320191.223183:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320191.223190:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320191.223195:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985363200:0@lo:38:kworker.0 00010000:00010000:1.0:1767320191.835470:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320192.863560:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320193.883578:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320194.907696:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320195.803975:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320195.804006:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985363840:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320195.804219:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320195.804386:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363840 00000100:00100000:0.0:1767320195.804396:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363840 00000100:00100000:2.0:1767320195.804447:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985363968:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320195.804462:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2500 00000100:00100000:0.0:1767320195.804476:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985363840:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320195.804487:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:0.0:1767320195.804509:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320195.804606:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985363840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 133us (387us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320195.804613:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2500 00000100:00100000:0.0:1767320195.804630:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985363968 00000100:00100000:0.0:1767320195.804634:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985363968 00000100:00100000:0.0:1767320195.804661:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2501 00000100:00100000:0.0:1767320195.804668:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985363968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320195.804683:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320195.804741:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985363968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (255us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320195.804747:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2501 00000100:00080000:3.0:1767320195.804943:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320195.804950:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320195.805947:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985363968:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320195.805966:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985363840:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320195.931950:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320196.266272:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320196.266523:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920603520 00000100:00100000:0.0:1767320196.266593:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920603520 00000100:00100000:0.0:1767320196.266729:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5075 00000100:00080000:3.0:1767320196.266730:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320196.266748:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920603520:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320196.266840:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920603520:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 102us (576us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320196.266847:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5075 00000100:00100000:2.0:1767320196.266981:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320196.268122:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920603392 00000100:00100000:1.0:1767320196.268132:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920603392 00000100:00100000:1.0:1767320196.268194:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5705 00000100:00100000:1.0:1767320196.268201:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920603392:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320196.268572:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320196.269014:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920603392:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 810us (2030us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320196.269023:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5705 00000100:00080000:2.0:1767320196.317138:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320196.317148:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320196.317152:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320196.317156:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320196.317161:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320196.317188:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5439 00000100:00080000:2.0:1767320196.317192:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320196.317199:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320196.317237:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985364096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320196.317267:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320196.317271:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320196.317273:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320196.317276:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5439 00000100:00080000:2.0:1767320196.317278:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320196.317282:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320196.317294:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518700 x1853163985364224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320196.317318:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320196.317322:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320196.317323:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320196.317327:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5439 00000100:00080000:2.0:1767320196.317329:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320196.317332:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320196.317344:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351bb80 x1853163985364480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320196.317367:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320196.317370:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320196.317372:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320196.317375:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5439 00000100:00080000:2.0:1767320196.317377:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320196.317380:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320196.317389:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5c00 x1853163985364736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320196.317627:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985364096:0@lo:38:kworker.0 00000100:00100000:1.0:1767320196.317703:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.317771:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985364224:0@lo:38:kworker.0 00000100:00100000:1.0:1767320196.317797:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.317824:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985364480:0@lo:38:kworker.0 00000100:00100000:1.0:1767320196.317846:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.317865:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985364736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320196.317899:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.317931:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985364352:0@lo:400:kworker.0 00000100:00100000:1.0:1767320196.317955:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.317974:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320196.317979:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985364608:0@lo:400:kworker.0 00000100:00100000:1.0:1767320196.317998:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.318024:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364096 00000100:00100000:1.0:1767320196.318031:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364096 00000100:00100000:1.0:1767320196.318056:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364480 00000100:00100000:1.0:1767320196.318059:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364480 00000100:00100000:1.0:1767320196.318068:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364736 00000100:00100000:1.0:1767320196.318070:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364736 00000100:00100000:1.0:1767320196.318076:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364352 00000100:00100000:1.0:1767320196.318078:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364352 00000100:00100000:1.0:1767320196.318101:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364608 00000100:00100000:1.0:1767320196.318103:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364608 00000100:00100000:1.0:1767320196.318116:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5076 00000100:00100000:1.0:1767320196.318122:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985364352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320196.318192:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985364352:12345-0@lo:400:kworker.0 Request processed in 69us (235us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320196.318202:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5076 00000100:00100000:1.0:1767320196.318218:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5077 00000100:00100000:1.0:1767320196.318224:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985364608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320196.318250:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985364608:12345-0@lo:400:kworker.0 Request processed in 27us (252us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320196.318256:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5077 00000100:00100000:1.0:1767320196.318266:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5706 00000100:00100000:1.0:1767320196.318270:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985364096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320196.318287:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320196.318314:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985364096:12345-0@lo:38:kworker.0 Request processed in 42us (620us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320196.318319:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5706 00000100:00100000:1.0:1767320196.318330:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5707 00000100:00100000:1.0:1767320196.318333:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985364480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320196.318342:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320196.318361:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985364480:12345-0@lo:38:kworker.0 Request processed in 26us (514us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320196.318367:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5707 00000100:00100000:1.0:1767320196.318377:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5708 00000100:00100000:1.0:1767320196.318381:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985364736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320196.318390:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320196.318410:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985364736:12345-0@lo:38:kworker.0 Request processed in 28us (510us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320196.318416:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5708 00000100:00100000:1.0:1767320196.318450:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985364608:0@lo:400:kworker.0 00000100:00100000:1.0:1767320196.318461:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985364352:0@lo:400:kworker.0 00000100:00080000:1.0:1767320196.318491:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320196.318496:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320196.318499:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985364096:0@lo:38:kworker.0 00000100:00080000:1.0:1767320196.318507:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320196.318510:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320196.318513:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985364480:0@lo:38:kworker.0 00000100:00080000:1.0:1767320196.318520:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320196.318523:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320196.318526:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985364736:0@lo:38:kworker.0 00000100:00100000:0.0:1767320196.318619:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364224 00000100:00100000:0.0:1767320196.318624:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364224 00000100:00100000:0.0:1767320196.318656:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1917 00000100:00100000:0.0:1767320196.318661:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985364224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320196.318672:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320196.318721:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985364224:12345-0@lo:38:kworker.0 Request processed in 59us (923us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320196.318727:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1917 00000100:00080000:0.0:1767320196.318752:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320196.318757:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320196.318759:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985364224:0@lo:38:kworker.0 00000100:00080000:3.0:1767320196.319352:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320196.956942:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320197.979351:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320199.004479:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320200.027409:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320200.925779:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985364864:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320200.925851:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320200.926261:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985364992:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320200.926293:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320200.926411:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364864 00000100:00100000:0.0:1767320200.926419:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364864 00000100:00100000:0.0:1767320200.926492:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985364992 00000100:00100000:0.0:1767320200.926495:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985364992 00000100:00100000:0.0:1767320200.926519:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2502 00000100:00100000:0.0:1767320200.926528:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985364864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320200.926559:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320200.926600:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985364864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (750us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320200.926618:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2502 00000100:00100000:0.0:1767320200.926635:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2503 00000100:00100000:0.0:1767320200.926642:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985364992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320200.926657:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320200.926696:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985364992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (402us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320200.926703:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2503 00000100:00100000:0.0:1767320200.926744:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985364992:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320200.926916:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985364864:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320200.927046:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320200.927050:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320201.052914:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320201.395681:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320201.395731:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320201.408502:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920603648 00000100:00100000:3.0:1767320201.408518:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920603648 00000100:00100000:3.0:1767320201.409076:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920603776 00000100:00100000:3.0:1767320201.409081:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920603776 00000100:00100000:3.0:1767320201.409153:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5078 00000100:00100000:3.0:1767320201.409185:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920603776:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320201.409278:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920603776:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 115us (13545us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320201.409287:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5078 00000100:00100000:3.0:1767320201.409306:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5709 00000100:00100000:3.0:1767320201.409311:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920603648:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320201.409331:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320201.409372:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920603648:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 59us (13701us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320201.409379:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5709 00000100:00080000:1.0:1767320201.410868:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320201.435337:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320201.435348:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320201.435351:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320201.435355:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320201.435360:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320201.435368:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5444 00000100:00080000:2.0:1767320201.435372:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320201.435380:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320201.435427:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4380 x1853163985365120/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320201.435452:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320201.435455:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320201.435457:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320201.435460:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5444 00000100:00080000:2.0:1767320201.435463:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320201.435466:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320201.435477:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4e00 x1853163985365248/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320201.435516:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320201.435520:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320201.435522:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320201.435525:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5444 00000100:00080000:2.0:1767320201.435527:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320201.435530:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320201.435541:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7100 x1853163985365504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320201.435557:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320201.435560:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320201.435562:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320201.435565:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5444 00000100:00080000:2.0:1767320201.435567:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320201.435570:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320201.435615:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7b80 x1853163985365760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320201.435738:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985365376:0@lo:400:kworker.0 00000100:00100000:1.0:1767320201.435804:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320201.435875:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320201.435881:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985365632:0@lo:400:kworker.0 00000100:00100000:1.0:1767320201.435916:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.436190:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985365376 00000100:00100000:2.0:1767320201.436199:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985365376 00000100:00100000:3.0:1767320201.436260:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985365120:0@lo:38:kworker.0 00000100:00100000:0.0:1767320201.436280:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985365632 00000100:00100000:0.0:1767320201.436286:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985365632 00000100:00100000:3.0:1767320201.436299:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320201.436332:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985365248:0@lo:38:kworker.0 00000100:00100000:3.0:1767320201.436363:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.436390:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985365120 00000100:00100000:2.0:1767320201.436396:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985365120 00000100:00100000:3.0:1767320201.436397:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985365504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320201.436423:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5079 00000100:00100000:3.0:1767320201.436427:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.436437:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985365376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320201.436448:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985365760:0@lo:38:kworker.0 00000100:00100000:3.0:1767320201.436469:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.436507:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985365376:12345-0@lo:400:kworker.0 Request processed in 72us (701us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320201.436516:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5079 00000100:00100000:2.0:1767320201.436537:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985365504 00000100:00100000:2.0:1767320201.436541:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985365504 00000100:00100000:2.0:1767320201.436556:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985365760 00000100:00100000:2.0:1767320201.436558:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985365760 00000100:00100000:3.0:1767320201.436562:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5710 00000100:00100000:3.0:1767320201.436569:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985365120:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320201.436576:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5711 00000100:00100000:2.0:1767320201.436583:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985365504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320201.436589:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767320201.436596:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320201.436640:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985365120:12345-0@lo:38:kworker.0 Request processed in 69us (340us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320201.436651:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5710 00000100:00100000:2.0:1767320201.436653:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985365504:12345-0@lo:38:kworker.0 Request processed in 65us (223us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320201.436662:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5711 00000100:00100000:3.0:1767320201.436669:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5712 00000100:00100000:3.0:1767320201.436676:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985365760:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320201.436698:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320201.436729:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.436735:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.436739:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985365120:0@lo:38:kworker.0 00000100:00100000:0.0:1767320201.437363:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5080 00000100:00100000:0.0:1767320201.437373:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985365632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320201.437496:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985365760:12345-0@lo:38:kworker.0 Request processed in 819us (1025us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320201.437504:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5712 00000100:00100000:3.0:1767320201.437554:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985365248 00000100:00100000:3.0:1767320201.437560:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985365248 00000100:00080000:1.0:1767320201.437570:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320201.437588:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1918 00000100:00100000:3.0:1767320201.437594:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985365248:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320201.437620:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320201.437625:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985365376:0@lo:400:kworker.0 00000100:00100000:3.0:1767320201.437655:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985365248:12345-0@lo:38:kworker.0 Request processed in 60us (1291us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320201.437663:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1918 00000100:00080000:2.0:1767320201.438020:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.438026:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.438029:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985365504:0@lo:38:kworker.0 00000100:00080000:2.0:1767320201.438039:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.438042:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.438044:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985365760:0@lo:38:kworker.0 00000100:00080000:2.0:1767320201.438062:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.438066:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.438068:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985365248:0@lo:38:kworker.0 00000100:00100000:0.0:1767320201.438149:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985365632:12345-0@lo:400:kworker.0 Request processed in 777us (2231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320201.438157:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5080 00000100:00100000:1.0:1767320201.438285:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985365632:0@lo:400:kworker.0 00010000:00010000:0.0:1767320202.076674:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320203.099633:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320204.123641:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320205.147398:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320206.044114:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320206.044557:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985365888:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320206.044627:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320206.044737:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985366016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320206.044761:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320206.044929:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985365888 00000100:00100000:3.0:1767320206.044938:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985365888 00000100:00100000:3.0:1767320206.044997:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366016 00000100:00100000:3.0:1767320206.045000:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366016 00000100:00100000:3.0:1767320206.045021:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2504 00000100:00100000:3.0:1767320206.045030:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985365888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320206.045056:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767320206.045190:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320206.045195:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320206.045238:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985365888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 209us (610us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320206.045247:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2504 00000100:00100000:3.0:1767320206.045264:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2505 00000100:00100000:3.0:1767320206.045335:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985366016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320206.045409:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320206.045436:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985366016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 166us (674us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320206.045442:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2505 00000100:00100000:0.0:1767320206.045572:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985366016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320206.045585:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985365888:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320206.171670:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320206.518147:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320206.518232:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320206.518369:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920603904 00000100:00100000:0.0:1767320206.518379:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920603904 00000100:00100000:0.0:1767320206.518417:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920604032 00000100:00100000:0.0:1767320206.518419:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920604032 00000100:00100000:0.0:1767320206.518451:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5081 00000100:00100000:0.0:1767320206.518460:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920604032:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320206.518536:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920604032:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (304us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320206.518543:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5081 00000100:00100000:0.0:1767320206.518561:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5713 00000100:00100000:0.0:1767320206.518565:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920603904:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320206.518583:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320206.518618:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920603904:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (481us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320206.518627:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5713 00000100:00080000:2.0:1767320206.519036:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320206.555926:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.555938:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320206.555941:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.555945:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.555948:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320206.555957:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.555963:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320206.555969:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320206.556006:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985366144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320206.556032:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.556036:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.556038:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320206.556041:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.556043:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320206.556047:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320206.556058:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519180 x1853163985366272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320206.556109:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.556113:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.556115:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320206.556119:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.556121:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320206.556124:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320206.556136:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4e00 x1853163985366528/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320206.556150:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.556153:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.556155:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320206.556158:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.556160:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320206.556185:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320206.556195:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4380 x1853163985366784/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320206.556332:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985366656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320206.556394:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320206.556462:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320206.556468:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985366400:0@lo:400:kworker.0 00000100:00100000:0.0:1767320206.556489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320206.556553:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366656 00000100:00100000:0.0:1767320206.556560:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366656 00000100:00100000:0.0:1767320206.556607:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366400 00000100:00100000:0.0:1767320206.556610:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366400 00000100:00100000:0.0:1767320206.556630:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5082 00000100:00100000:0.0:1767320206.556638:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985366656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320206.556682:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985366656:12345-0@lo:400:kworker.0 Request processed in 45us (287us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320206.556701:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5082 00000100:00100000:0.0:1767320206.556718:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5083 00000100:00100000:0.0:1767320206.556724:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985366400:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320206.556750:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985366400:12345-0@lo:400:kworker.0 Request processed in 27us (261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320206.556757:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5083 00000100:00080000:0.0:1767320206.556772:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320206.556801:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985366400:0@lo:400:kworker.0 00000100:00100000:0.0:1767320206.556811:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985366656:0@lo:400:kworker.0 00000100:00100000:3.0:1767320206.557739:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985366144:0@lo:38:kworker.0 00000100:00100000:3.0:1767320206.557785:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320206.557825:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985366272:0@lo:38:kworker.0 00000100:00100000:3.0:1767320206.557850:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320206.557883:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985366528:0@lo:38:kworker.0 00000100:00100000:3.0:1767320206.557903:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320206.557917:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985366784:0@lo:38:kworker.0 00000100:00100000:3.0:1767320206.557935:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320206.558005:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366272 00000100:00100000:2.0:1767320206.558012:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366272 00000100:00100000:2.0:1767320206.558045:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1919 00000100:00100000:2.0:1767320206.558052:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985366272:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320206.558071:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320206.558115:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985366272:12345-0@lo:38:kworker.0 Request processed in 63us (264us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320206.558121:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1919 00000100:00100000:1.0:1767320206.558516:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366144 00000100:00100000:1.0:1767320206.558521:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366144 00000100:00100000:1.0:1767320206.558549:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366528 00000100:00100000:1.0:1767320206.558552:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366528 00000100:00100000:1.0:1767320206.558565:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5714 00000100:00100000:1.0:1767320206.558569:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985366144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320206.558580:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320206.558611:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985366144:12345-0@lo:38:kworker.0 Request processed in 39us (825us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320206.558617:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5714 00000100:00100000:1.0:1767320206.558628:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366784 00000100:00100000:1.0:1767320206.558631:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366784 00000100:00100000:1.0:1767320206.558641:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5715 00000100:00100000:1.0:1767320206.558645:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985366528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320206.558655:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320206.558677:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985366528:12345-0@lo:38:kworker.0 Request processed in 30us (772us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320206.558683:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5715 00000100:00100000:1.0:1767320206.558693:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5716 00000100:00100000:1.0:1767320206.558697:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985366784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320206.558705:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320206.558728:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985366784:12345-0@lo:38:kworker.0 Request processed in 30us (792us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320206.558734:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5716 00000100:00080000:3.0:1767320206.559902:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320206.559911:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320206.559915:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985366144:0@lo:38:kworker.0 00000100:00080000:3.0:1767320206.559928:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320206.559931:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320206.559934:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985366272:0@lo:38:kworker.0 00000100:00080000:3.0:1767320206.559943:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320206.559946:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320206.559949:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985366528:0@lo:38:kworker.0 00000100:00080000:3.0:1767320206.559958:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320206.559961:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320206.559964:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985366784:0@lo:38:kworker.0 00010000:00010000:1.0:1767320207.195970:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320208.219510:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320209.245323:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320210.271220:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320211.163568:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320211.163592:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985366912:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320211.163662:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320211.163846:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985366912 00000100:00100000:2.0:1767320211.163854:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985366912 00000100:00100000:2.0:1767320211.163909:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2506 00000100:00100000:2.0:1767320211.163919:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985366912:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320211.163946:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320211.164004:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5ba6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985366912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (334us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320211.164012:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2506 00000100:00080000:0.0:1767320211.164268:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320211.164301:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985366912:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320211.164842:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985367040:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320211.164894:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320211.165897:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367040 00000100:00100000:2.0:1767320211.165905:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367040 00000100:00100000:2.0:1767320211.165967:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2507 00000100:00100000:2.0:1767320211.165976:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985367040:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320211.166080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:2.0:1767320211.166926:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320211.167030:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985367040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1048us (2127us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320211.167039:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2507 00000100:00100000:0.0:1767320211.167086:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985367040:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320211.294582:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320211.626585:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320211.626744:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320211.627015:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920604160 00000100:00100000:0.0:1767320211.627025:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920604160 00000100:00100000:0.0:1767320211.627105:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920604288 00000100:00100000:0.0:1767320211.627109:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920604288 00000100:00080000:3.0:1767320211.627159:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320211.628770:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5084 00000100:00100000:2.0:1767320211.628793:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920604288:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320211.628881:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920604288:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (2137us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320211.630653:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5717 00000100:00100000:1.0:1767320211.630665:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920604160:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320211.630690:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320211.630713:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5084 00000100:00100000:1.0:1767320211.630755:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920604160:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 89us (4177us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320211.630763:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5717 00000100:00080000:1.0:1767320211.675285:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320211.675298:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320211.675302:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320211.675306:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320211.675311:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320211.675319:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5454 00000100:00080000:1.0:1767320211.675324:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320211.675333:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320211.675371:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5c00 x1853163985367168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320211.675388:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320211.675391:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320211.675393:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320211.675397:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5454 00000100:00080000:1.0:1767320211.675399:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320211.675403:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320211.675427:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4380 x1853163985367296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320211.675458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985367168:0@lo:38:kworker.0 00000100:00080000:1.0:1767320211.675465:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320211.675470:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320211.675473:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320211.675477:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5454 00000100:00080000:1.0:1767320211.675480:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320211.675484:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320211.675498:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7100 x1853163985367552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320211.675520:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320211.675523:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320211.675526:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320211.675529:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320211.675533:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5454 00000100:00080000:1.0:1767320211.675537:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320211.675540:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320211.675552:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939500 x1853163985367808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320211.675579:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985367424:0@lo:400:kworker.0 00000100:00100000:2.0:1767320211.675595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985367296:0@lo:38:kworker.0 00000100:00100000:3.0:1767320211.675605:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367168 00000100:00100000:3.0:1767320211.675614:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367168 00000100:00100000:1.0:1767320211.675619:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320211.675631:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320211.675645:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367424 00000100:00100000:1.0:1767320211.675648:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320211.675649:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367424 00000100:00100000:1.0:1767320211.675660:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985367680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320211.675685:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985367552:0@lo:38:kworker.0 00000100:00100000:3.0:1767320211.675691:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5085 00000100:00100000:1.0:1767320211.675692:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320211.675703:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985367424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320211.675724:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320211.675755:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985367808:0@lo:38:kworker.0 00000100:00100000:3.0:1767320211.675768:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985367424:12345-0@lo:400:kworker.0 Request processed in 66us (151us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320211.675781:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5085 00000100:00100000:2.0:1767320211.675782:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320211.675833:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367680 00000100:00100000:1.0:1767320211.675839:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367680 00000100:00100000:1.0:1767320211.675865:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367552 00000100:00100000:1.0:1767320211.675868:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367552 00000100:00100000:1.0:1767320211.675882:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5086 00000100:00100000:1.0:1767320211.675889:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985367680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320211.675926:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985367680:12345-0@lo:400:kworker.0 Request processed in 37us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320211.675933:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5718 00000100:00100000:0.0:1767320211.675940:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985367168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320211.675961:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320211.675997:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985367168:12345-0@lo:38:kworker.0 Request processed in 56us (471us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320211.676004:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5086 00000100:00100000:0.0:1767320211.676015:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5718 00000100:00100000:1.0:1767320211.676027:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5719 00000100:00100000:1.0:1767320211.676033:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985367552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320211.676044:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320211.676104:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985367680:0@lo:400:kworker.0 00000100:00100000:0.0:1767320211.676122:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985367424:0@lo:400:kworker.0 00000100:00080000:0.0:1767320211.676157:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320211.676200:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367296 00000100:00100000:0.0:1767320211.676204:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367296 00000100:00100000:0.0:1767320211.676227:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1920 00000100:00100000:0.0:1767320211.676231:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985367296:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320211.676242:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320211.676621:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367808 00000100:00100000:3.0:1767320211.676626:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367808 00000100:00100000:3.0:1767320211.676646:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5720 00000100:00100000:3.0:1767320211.676651:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985367808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320211.676662:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320211.679767:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320211.679775:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985367808:12345-0@lo:38:kworker.0 Request processed in 3039us (3913us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767320211.679778:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320211.679785:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985367168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320211.679794:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5720 00000100:00080000:2.0:1767320211.679828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320211.679834:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320211.679839:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985367552:0@lo:38:kworker.0 00000100:00080000:2.0:1767320211.679849:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320211.679854:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320211.679857:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985367808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320211.680001:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985367552:12345-0@lo:38:kworker.0 Request processed in 3964us (4274us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320211.680009:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5719 00000100:00100000:0.0:1767320211.680182:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985367296:12345-0@lo:38:kworker.0 Request processed in 3924us (4526us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320211.680192:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1920 00000100:00080000:2.0:1767320211.680639:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320211.680644:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320211.680648:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985367296:0@lo:38:kworker.0 00010000:00010000:2.0:1767320212.316330:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320213.344527:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320214.365980:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320215.387433:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320215.723827:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320215.723928:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920604672 00000100:00100000:1.0:1767320215.723940:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920604672 00000100:00100000:1.0:1767320215.724008:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5087 00000100:00100000:1.0:1767320215.724018:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920604672:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320215.724101:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920604672:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320215.724109:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5087 00000400:00000100:2.0:1767320215.724468:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920604416 offset 0 length 520: 4 00000100:00100000:2.0:1767320215.724515:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:0.0:1767320215.725688:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320215.725806:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920604544 00000100:00100000:3.0:1767320215.725816:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920604544 00000100:00100000:3.0:1767320215.725863:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5721 00000100:00100000:3.0:1767320215.725870:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920604544:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320215.725892:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320215.725997:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920604544:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 125us (1488us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320215.726005:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5721 00000100:00100000:0.0:1767320216.283479:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320216.283503:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985367936:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320216.283586:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320216.283833:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985367936 00000100:00100000:3.0:1767320216.283843:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985367936 00000100:00100000:3.0:1767320216.283902:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2508 00000100:00100000:3.0:1767320216.283911:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985367936:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320216.283940:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320216.283987:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985367936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (400us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320216.283995:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2508 00000100:00080000:0.0:1767320216.284062:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320216.284097:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985367936:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320216.284473:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985368064:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320216.284500:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320216.284633:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985368064 00000100:00100000:3.0:1767320216.284637:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985368064 00000100:00100000:3.0:1767320216.284675:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2509 00000100:00100000:3.0:1767320216.284690:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985368064:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320216.284706:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320216.284734:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985368064:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (232us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320216.284739:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2509 00000100:00080000:0.0:1767320216.284888:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320216.284909:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985368064:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320216.411397:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320216.795285:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320191/real 1767320191] req@ffff8ce1a54b6a00 x1853163985362688/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320216 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320216.795329:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a54b6a00 x1853163985362688/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320216 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320216.795350:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320216.797952:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320216.797962:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320216.797966:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320216.797973:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5439 00000100:00080000:3.0:1767320216.797978:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320216.797985:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320216.798023:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985368192/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320216.798040:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320216.798048:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320216.798050:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320216.798054:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320216.798056:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320216.798059:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320216.798072:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519180 x1853163985368320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320216.798080:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320216.798083:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320216.798085:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320216.798088:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320216.798090:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320216.798093:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320216.798104:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2680 x1853163985368448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320216.798210:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985368576:0@lo:400:kworker.0 00000100:00080000:2.0:1767320216.798230:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320216.798235:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985362688:0@lo:250:kworker.0 00000100:00100000:2.0:1767320216.798261:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985368192:0@lo:250:kworker.0 00000100:00100000:0.0:1767320216.798279:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000100:2.0:1767320216.798280:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985368192 offset 0 length 520: 4 00000100:00100000:2.0:1767320216.798327:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985368320:0@lo:38:kworker.0 00000100:00080000:3.0:1767320216.798342:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320216.798346:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320216.798348:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320216.798349:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320216.798352:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320216.798354:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320216.798358:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320216.798364:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985368448:0@lo:38:kworker.0 00000100:00100000:3.0:1767320216.798382:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3480 x1853163985368704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320216.798385:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320216.798413:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985368704:0@lo:38:kworker.0 00000100:00080000:3.0:1767320216.798422:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320216.798425:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320216.798428:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320216.798430:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320216.798442:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320216.798445:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320216.798448:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320216.798460:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985368576 00000100:00100000:3.0:1767320216.798460:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05f80 x1853163985368960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320216.798466:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985368576 00000100:00100000:2.0:1767320216.798523:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985368320 00000100:00100000:2.0:1767320216.798527:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985368320 00000100:00100000:2.0:1767320216.798538:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985368704 00000100:00100000:2.0:1767320216.798540:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985368704 00000100:00100000:2.0:1767320216.798553:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5088 00000100:00100000:2.0:1767320216.798561:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985368576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.798611:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985368576:12345-0@lo:400:kworker.0 Request processed in 42us (320us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320216.798618:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5088 00000100:00100000:2.0:1767320216.798634:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5722 00000100:00100000:2.0:1767320216.798638:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985368320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320216.798655:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320216.798678:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985368320:12345-0@lo:38:kworker.0 Request processed in 39us (328us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320216.798685:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5722 00000100:00100000:2.0:1767320216.798694:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5723 00000100:00100000:2.0:1767320216.798698:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985368704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320216.798706:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320216.798722:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985368704:12345-0@lo:38:kworker.0 Request processed in 23us (291us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320216.798726:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5723 00000100:00100000:2.0:1767320216.798741:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985368960:0@lo:38:kworker.0 00000100:00100000:2.0:1767320216.798760:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320216.798783:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320216.798787:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320216.798790:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985368704:0@lo:38:kworker.0 00000100:00080000:2.0:1767320216.798796:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320216.798799:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320216.798802:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985368320:0@lo:38:kworker.0 00000100:00100000:2.0:1767320216.798820:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985368960 00000100:00100000:2.0:1767320216.798823:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985368960 00000100:00100000:2.0:1767320216.798834:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5724 00000100:00100000:2.0:1767320216.798838:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985368960:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320216.798846:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320216.798868:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985368960:12345-0@lo:38:kworker.0 Request processed in 29us (108us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320216.798873:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5724 00000100:00080000:2.0:1767320216.798890:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320216.798894:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320216.798897:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985368960:0@lo:38:kworker.0 00000100:00080000:2.0:1767320216.798966:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320216.798990:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985368832:0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799011:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320216.799032:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985368832 00000100:00100000:2.0:1767320216.799036:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985368832 00000100:00100000:2.0:1767320216.799058:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5089 00000100:00100000:2.0:1767320216.799063:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985368832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799089:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985368832:12345-0@lo:400:kworker.0 Request processed in 26us (77us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320216.799095:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5089 00000100:00080000:2.0:1767320216.799107:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320216.799122:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985368832:0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799134:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985369216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799157:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320216.799193:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320216.799198:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985369088:0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799215:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320216.799236:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985368448 00000100:00100000:2.0:1767320216.799239:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985368448 00000100:00100000:2.0:1767320216.799255:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1921 00000100:00100000:2.0:1767320216.799259:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985368448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320216.799269:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320216.799292:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985368448:12345-0@lo:38:kworker.0 Request processed in 32us (906us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320216.799296:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1921 00000100:00080000:2.0:1767320216.799325:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320216.799328:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320216.799331:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985368448:0@lo:38:kworker.0 00000100:00100000:2.0:1767320216.799420:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369216 00000100:00100000:2.0:1767320216.799424:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369216 00000100:00100000:2.0:1767320216.799445:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369088 00000100:00100000:2.0:1767320216.799447:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369088 00000100:00100000:2.0:1767320216.799459:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3313 00000100:00100000:2.0:1767320216.799464:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985369216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799493:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985369216:12345-0@lo:400:kworker.0 Request processed in 29us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320216.799499:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3313 00000100:00100000:2.0:1767320216.799509:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3314 00000100:00100000:2.0:1767320216.799514:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985369088:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799533:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985369088:12345-0@lo:400:kworker.0 Request processed in 19us (316us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320216.799539:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3314 00000100:00100000:2.0:1767320216.799557:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985369088:0@lo:400:kworker.0 00000100:00100000:2.0:1767320216.799565:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985369216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320216.800772:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985368576:0@lo:400:kworker.0 00010000:00010000:1.0:1767320217.443047:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320218.459573:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320219.485537:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320220.507457:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320220.851770:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320220.852865:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320220.853283:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920604928 00000100:00100000:0.0:1767320220.853298:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920604928 00000100:00100000:0.0:1767320220.853402:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920604800 00000100:00100000:0.0:1767320220.853405:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920604800 00000100:00100000:0.0:1767320220.853431:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5090 00000100:00100000:0.0:1767320220.853444:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920604928:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320220.853559:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920604928:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 118us (1806us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320220.853568:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5090 00000100:00100000:0.0:1767320220.853591:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5725 00000100:00100000:0.0:1767320220.853596:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920604800:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320220.853621:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320220.853659:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920604800:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 61us (804us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320220.853666:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5725 00000100:00080000:2.0:1767320220.855365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320221.407475:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985369344:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320221.407476:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985369472:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320221.407592:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320221.407645:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369344 00000100:00100000:1.0:1767320221.407656:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369344 00000100:00100000:1.0:1767320221.407736:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2510 00000100:00100000:1.0:1767320221.407748:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985369344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320221.407787:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320221.407860:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985369344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 114us (268us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320221.407869:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2510 00000100:00080000:3.0:1767320221.407919:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320221.408137:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985369344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320221.408708:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320221.408946:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369472 00000100:00100000:1.0:1767320221.408954:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369472 00000100:00100000:1.0:1767320221.409022:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2511 00000100:00100000:1.0:1767320221.409032:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985369472:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320221.409059:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320221.409130:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985369472:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (423us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320221.409138:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2511 00000100:00100000:2.0:1767320221.409314:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985369472:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320221.409553:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767320221.531981:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320221.915532:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320221.915546:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320221.915550:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320221.915555:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320221.915560:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320221.915571:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5465 00000100:00080000:2.0:1767320221.915577:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320221.915587:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320221.915643:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2d80 x1853163985369600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320221.915755:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320221.915759:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320221.915762:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320221.915766:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5465 00000100:00080000:2.0:1767320221.915769:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320221.915773:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320221.915790:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3480 x1853163985369728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320221.915908:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985369856:0@lo:400:kworker.0 00000100:00100000:1.0:1767320221.916010:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320221.916131:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985369600:0@lo:38:kworker.0 00000100:00100000:0.0:1767320221.916191:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320221.916215:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985369728:0@lo:38:kworker.0 00000100:00100000:0.0:1767320221.916252:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320221.916312:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369728 00000100:00100000:0.0:1767320221.916321:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369728 00000100:00100000:0.0:1767320221.916360:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1922 00000100:00100000:0.0:1767320221.916369:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985369728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320221.916392:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320221.916436:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985369728:12345-0@lo:38:kworker.0 Request processed in 67us (183us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320221.916445:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1922 00000100:00100000:0.0:1767320221.916472:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369856 00000100:00100000:0.0:1767320221.916476:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369856 00000100:00100000:0.0:1767320221.916520:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369600 00000100:00100000:0.0:1767320221.916524:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369600 00000100:00100000:0.0:1767320221.916541:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5091 00000100:00100000:0.0:1767320221.916548:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985369856:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320221.916626:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320221.916630:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320221.916632:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320221.916636:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5465 00000100:00080000:2.0:1767320221.916639:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320221.916643:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320221.916674:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99f80 x1853163985369984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320221.916953:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5726 00000100:00100000:3.0:1767320221.916960:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985369600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320221.916971:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320221.917018:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985369600:12345-0@lo:38:kworker.0 Request processed in 57us (853us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320221.917024:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5726 00000100:00080000:3.0:1767320221.917046:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320221.917077:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985370112:0@lo:400:kworker.0 00000100:00100000:3.0:1767320221.917107:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320221.917135:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985370112 00000100:00100000:3.0:1767320221.917139:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985370112 00000100:00100000:3.0:1767320221.917187:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5092 00000100:00100000:3.0:1767320221.917194:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985370112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320221.917236:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985370112:12345-0@lo:400:kworker.0 Request processed in 42us (128us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320221.917243:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5092 00000100:00100000:3.0:1767320221.917285:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985370112:0@lo:400:kworker.0 00000100:00080000:3.0:1767320221.917306:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320221.917339:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985369984:0@lo:38:kworker.0 00000100:00100000:3.0:1767320221.917381:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320221.917405:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320221.917410:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320221.917413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985369600:0@lo:38:kworker.0 00000100:00080000:3.0:1767320221.917422:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320221.917425:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320221.917428:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985369728:0@lo:38:kworker.0 00000100:00100000:3.0:1767320221.917453:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985369984 00000100:00100000:3.0:1767320221.917457:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985369984 00000100:00100000:3.0:1767320221.917475:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5727 00000100:00100000:3.0:1767320221.917479:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985369984:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320221.917490:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320221.917519:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985369984:12345-0@lo:38:kworker.0 Request processed in 39us (137us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320221.917525:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5727 00000100:00080000:3.0:1767320221.917549:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320221.917553:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320221.917556:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985369984:0@lo:38:kworker.0 00000100:00100000:0.0:1767320221.918217:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985369856:12345-0@lo:400:kworker.0 Request processed in 1666us (2204us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320221.918235:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5091 00000100:00100000:1.0:1767320221.918360:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985369856:0@lo:400:kworker.0 00000100:00080000:2.0:1767320221.918472:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320221.918477:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320221.918480:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320221.918485:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5465 00000100:00080000:2.0:1767320221.918489:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320221.918494:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320221.918522:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9b800 x1853163985370240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320221.918576:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985370240:0@lo:38:kworker.0 00000100:00100000:3.0:1767320221.918614:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320221.918790:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985370368:0@lo:400:kworker.0 00000100:00100000:1.0:1767320221.918824:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320221.919016:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985370368 00000100:00100000:3.0:1767320221.919023:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985370368 00000100:00100000:3.0:1767320221.919066:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3315 00000100:00100000:3.0:1767320221.919074:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985370368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320221.919303:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985370240 00000100:00100000:0.0:1767320221.919307:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985370240 00000100:00100000:0.0:1767320221.919335:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5728 00000100:00100000:0.0:1767320221.919341:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985370240:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320221.919361:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320221.919401:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985370240:12345-0@lo:38:kworker.0 Request processed in 59us (786us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320221.919407:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5728 00000100:00080000:0.0:1767320221.919440:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320221.919444:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320221.919447:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985370240:0@lo:38:kworker.0 00000100:00100000:0.0:1767320221.919479:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985370496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320221.919506:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320221.919544:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985370496 00000100:00100000:0.0:1767320221.919549:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985370496 00000100:00100000:0.0:1767320221.919574:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3316 00000100:00100000:0.0:1767320221.919579:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985370496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320221.919613:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985370496:12345-0@lo:400:kworker.0 Request processed in 34us (107us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320221.919619:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3316 00000100:00100000:0.0:1767320221.919644:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985370496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320221.920321:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985370368:12345-0@lo:400:kworker.0 Request processed in 1243us (1492us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320221.920328:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3315 00000100:00100000:1.0:1767320221.920422:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985370368:0@lo:400:kworker.0 00010000:00010000:1.0:1767320222.561098:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320223.579626:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320224.603827:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320225.627519:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320225.973934:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320225.974382:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320225.974919:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920605184 00000100:00100000:1.0:1767320225.974933:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920605184 00000100:00100000:1.0:1767320225.975022:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920605056 00000100:00100000:1.0:1767320225.975025:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920605056 00000100:00100000:1.0:1767320225.975047:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5093 00000100:00100000:1.0:1767320225.975056:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920605184:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767320225.975190:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320225.975224:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5729 00000100:00100000:3.0:1767320225.975230:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920605056:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320225.975254:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320225.975342:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920605056:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 111us (1355us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320225.975351:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5729 00000100:00100000:1.0:1767320225.975738:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920605184:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 682us (1817us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320225.975745:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5093 00000100:00100000:1.0:1767320226.523568:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985370624:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320226.523665:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320226.523931:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320226.523941:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985370752:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320226.523970:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320226.524059:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985370624 00000100:00100000:3.0:1767320226.524071:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985370624 00000100:00100000:3.0:1767320226.524153:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985370752 00000100:00100000:3.0:1767320226.524156:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985370752 00000100:00100000:3.0:1767320226.524211:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2512 00000100:00100000:3.0:1767320226.524221:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985370624:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320226.524259:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320226.524325:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985370624:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (660us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320226.524333:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2512 00000100:00100000:3.0:1767320226.524353:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2513 00000100:00100000:3.0:1767320226.524360:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985370752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320226.524377:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320226.524404:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985370752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (434us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320226.524410:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2513 00000100:00080000:1.0:1767320226.524490:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320226.524496:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320226.524535:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985370752:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320226.524549:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985370624:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320226.651518:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320227.035368:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320227.035383:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320227.035386:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320227.035393:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320227.035398:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320227.035409:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5470 00000100:00080000:2.0:1767320227.035415:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320227.035423:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320227.035474:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9b480 x1853163985370880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320227.035501:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320227.035505:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320227.035507:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320227.035510:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5470 00000100:00080000:2.0:1767320227.035513:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320227.035517:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320227.035530:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98000 x1853163985371008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320227.035630:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320227.035660:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985370880:0@lo:38:kworker.0 00000100:00100000:0.0:1767320227.035753:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320227.035754:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320227.035759:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320227.035761:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320227.035765:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5470 00000100:00080000:2.0:1767320227.035768:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320227.035772:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320227.035786:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9b100 x1853163985371264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320227.035826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985371008:0@lo:38:kworker.0 00000100:00100000:1.0:1767320227.035851:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985370880 00000100:00100000:0.0:1767320227.035855:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320227.035861:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985370880 00000100:00100000:0.0:1767320227.035897:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985371264:0@lo:38:kworker.0 00000100:00100000:1.0:1767320227.035906:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5730 00000100:00100000:1.0:1767320227.035915:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985370880:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320227.035918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320227.035951:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371008 00010000:00020000:1.0:1767320227.035952:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320227.035954:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371008 00000100:00100000:0.0:1767320227.035975:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1923 00000100:00100000:0.0:1767320227.035980:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985371008:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320227.035990:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320227.036000:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985370880:12345-0@lo:38:kworker.0 Request processed in 86us (247us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320227.036008:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5730 00000100:00100000:0.0:1767320227.036011:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985371008:12345-0@lo:38:kworker.0 Request processed in 31us (156us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320227.036015:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1923 00000100:00100000:1.0:1767320227.036023:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371264 00000100:00100000:1.0:1767320227.036027:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371264 00000100:00100000:1.0:1767320227.036041:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5731 00000100:00100000:1.0:1767320227.036045:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985371264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320227.036054:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320227.036076:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985371264:12345-0@lo:38:kworker.0 Request processed in 30us (157us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320227.036082:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5731 00000100:00080000:0.0:1767320227.036084:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320227.036088:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320227.036092:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985370880:0@lo:38:kworker.0 00000100:00080000:0.0:1767320227.036101:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320227.036104:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320227.036108:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985371008:0@lo:38:kworker.0 00000100:00080000:0.0:1767320227.036156:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320227.036160:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320227.036184:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985371264:0@lo:38:kworker.0 00000100:00100000:3.0:1767320227.036799:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985371392:0@lo:400:kworker.0 00000100:00100000:3.0:1767320227.036903:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320227.036989:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371392 00000100:00100000:1.0:1767320227.036992:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371392 00000100:00100000:1.0:1767320227.037041:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5094 00000100:00100000:1.0:1767320227.037050:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985371392:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320227.037088:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320227.037175:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320227.037179:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320227.037182:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320227.037186:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5470 00000100:00080000:2.0:1767320227.037198:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320227.037203:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320227.037224:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98380 x1853163985371520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320227.037283:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985371776:0@lo:400:kworker.0 00000100:00100000:0.0:1767320227.037341:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985371520:0@lo:38:kworker.0 00000100:00100000:1.0:1767320227.038440:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985371392:12345-0@lo:400:kworker.0 Request processed in 1386us (1531us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320227.038452:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5094 00000100:00100000:1.0:1767320227.038499:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985371648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320227.038540:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320227.039084:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371648 00000100:00100000:1.0:1767320227.039092:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371648 00000100:00100000:1.0:1767320227.039483:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3317 00000100:00100000:1.0:1767320227.039492:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985371648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320227.039558:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985371648:12345-0@lo:400:kworker.0 Request processed in 66us (1017us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320227.039566:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3317 00000100:00100000:3.0:1767320227.039784:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985371392:0@lo:400:kworker.0 00000100:00100000:3.0:1767320227.039794:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985371136:0@lo:400:kworker.0 00000100:00100000:1.0:1767320227.040029:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985371648:0@lo:400:kworker.0 00000100:00100000:2.0:1767320227.040989:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320227.041136:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320227.041155:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371776 00000100:00100000:1.0:1767320227.041175:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371776 00000100:00100000:2.0:1767320227.041434:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371520 00000100:00100000:2.0:1767320227.041438:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371520 00000100:00100000:2.0:1767320227.041480:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5732 00000100:00100000:2.0:1767320227.041490:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985371520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320227.041515:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320227.041570:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985371520:12345-0@lo:38:kworker.0 Request processed in 80us (434us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320227.041577:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5732 00000100:00080000:0.0:1767320227.041613:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320227.041620:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320227.041624:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985371520:0@lo:38:kworker.0 00000100:00100000:1.0:1767320227.041815:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3318 00000100:00100000:1.0:1767320227.041832:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985371776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320227.041888:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985371776:12345-0@lo:400:kworker.0 Request processed in 67us (900us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320227.041895:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3318 00000100:00100000:2.0:1767320227.042072:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985371776:0@lo:400:kworker.0 00000100:00100000:3.0:1767320227.044362:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320227.044541:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371136 00000100:00100000:2.0:1767320227.044551:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371136 00000100:00080000:0.0:1767320227.044767:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320227.044913:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5095 00000100:00100000:2.0:1767320227.045037:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985371136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320227.050065:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985371136:12345-0@lo:400:kworker.0 Request processed in 5128us (5694us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320227.050085:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5095 00000100:00100000:3.0:1767320227.050511:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985371136:0@lo:400:kworker.0 00010000:00010000:0.0:1767320227.675572:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320228.700241:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320229.723926:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320230.758278:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320231.084523:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320231.084613:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320231.084706:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920605440 00000100:00100000:2.0:1767320231.084716:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920605440 00000100:00100000:2.0:1767320231.084791:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920605312 00000100:00100000:2.0:1767320231.084795:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920605312 00000100:00100000:2.0:1767320231.084812:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5096 00000100:00100000:2.0:1767320231.084823:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920605440:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320231.084907:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920605440:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (395us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320231.084915:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5096 00000100:00100000:2.0:1767320231.084934:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5733 00000100:00100000:2.0:1767320231.084938:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920605312:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320231.084957:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320231.084982:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920605312:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (369us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320231.084989:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5733 00000100:00080000:1.0:1767320231.085240:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320231.647594:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320231.647617:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985371904:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320231.647696:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320231.647821:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985371904 00000100:00100000:3.0:1767320231.647830:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985371904 00000100:00100000:3.0:1767320231.647890:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2514 00000100:00100000:3.0:1767320231.647899:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985371904:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320231.647929:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320231.647973:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985371904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (277us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320231.647981:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2514 00000100:00080000:0.0:1767320231.648084:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320231.648133:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985371904:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320231.648505:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320231.648517:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985372032:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320231.648552:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320231.648631:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372032 00000100:00100000:3.0:1767320231.648636:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372032 00000100:00100000:3.0:1767320231.648689:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2515 00000100:00100000:3.0:1767320231.648696:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985372032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320231.648714:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320231.648746:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985372032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 51us (193us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320231.648752:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2515 00000100:00080000:1.0:1767320231.648806:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320231.648830:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985372032:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320231.771752:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320232.155753:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320232.155763:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320232.155767:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320232.155770:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320232.155774:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320232.155782:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5475 00000100:00080000:0.0:1767320232.155786:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320232.155794:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320232.155829:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bd180 x1853163985372160/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320232.155856:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320232.155859:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320232.155861:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320232.155864:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5475 00000100:00080000:0.0:1767320232.155866:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320232.155869:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320232.155892:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bf100 x1853163985372288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320232.155942:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320232.155945:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320232.155947:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320232.155951:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5475 00000100:00080000:0.0:1767320232.155953:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320232.155956:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320232.155966:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bca80 x1853163985372544/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320232.155981:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320232.155984:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320232.155986:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320232.155989:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5475 00000100:00080000:0.0:1767320232.155991:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320232.155994:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320232.156003:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bea00 x1853163985372800/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320232.156084:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985373056:0@lo:400:kworker.0 00000100:00100000:3.0:1767320232.156095:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985372672:0@lo:400:kworker.0 00000100:00100000:1.0:1767320232.156190:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985372416:0@lo:400:kworker.0 00000100:00100000:2.0:1767320232.156244:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985372160:0@lo:38:kworker.0 00000100:00100000:3.0:1767320232.159638:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320232.159758:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372672 00000100:00100000:3.0:1767320232.159778:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372672 00000100:00100000:3.0:1767320232.159839:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5097 00000100:00100000:3.0:1767320232.159851:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985372672:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320232.159913:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985372672:12345-0@lo:400:kworker.0 Request processed in 66us (285us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320232.159920:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5097 00000100:00100000:3.0:1767320232.159969:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985372672:0@lo:400:kworker.0 00000100:00100000:3.0:1767320232.160012:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985372928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320232.160071:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320232.160114:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372928 00000100:00100000:3.0:1767320232.160117:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372928 00000100:00100000:3.0:1767320232.160143:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3319 00000100:00100000:3.0:1767320232.160150:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985372928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320232.160203:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985372928:12345-0@lo:400:kworker.0 Request processed in 54us (131us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320232.160210:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3319 00000100:00100000:3.0:1767320232.160233:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985372928:0@lo:400:kworker.0 00000100:00080000:3.0:1767320232.160256:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320232.161095:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320232.162604:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373056 00000100:00100000:3.0:1767320232.162617:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373056 00000100:00100000:3.0:1767320232.162682:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3320 00000100:00100000:3.0:1767320232.162692:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985373056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320232.162771:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985373056:12345-0@lo:400:kworker.0 Request processed in 81us (1678us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320232.162778:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3320 00000100:00100000:1.0:1767320232.163203:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320232.163442:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372416 00000100:00100000:3.0:1767320232.163448:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372416 00000100:00100000:3.0:1767320232.163592:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5098 00000100:00100000:3.0:1767320232.163600:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985372416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320232.163649:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985372416:12345-0@lo:400:kworker.0 Request processed in 50us (447us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320232.163652:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320232.163659:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5098 00000100:00100000:1.0:1767320232.163693:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985372416:0@lo:400:kworker.0 00000100:00100000:0.0:1767320232.163739:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985373056:0@lo:400:kworker.0 00000100:00100000:2.0:1767320232.163830:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320232.163861:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985372288:0@lo:38:kworker.0 00000100:00100000:2.0:1767320232.164058:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320232.164090:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985372544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320232.164111:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320232.164136:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985372800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320232.164156:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320232.164189:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372160 00000100:00100000:1.0:1767320232.164195:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372160 00000100:00100000:1.0:1767320232.164219:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372544 00000100:00100000:1.0:1767320232.164222:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372544 00000100:00100000:0.0:1767320232.164256:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372288 00000100:00100000:0.0:1767320232.164271:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372288 00000100:00100000:0.0:1767320232.164294:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1924 00000100:00100000:0.0:1767320232.164299:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985372288:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320232.164580:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320232.164624:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985372288:12345-0@lo:38:kworker.0 Request processed in 323us (565us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320232.164631:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1924 00000100:00100000:2.0:1767320232.164779:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985372800 00000100:00100000:2.0:1767320232.164783:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985372800 00000100:00100000:2.0:1767320232.164808:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5734 00000100:00100000:2.0:1767320232.164812:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985372160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320232.164823:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320232.164849:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985372160:12345-0@lo:38:kworker.0 Request processed in 36us (1020us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320232.164855:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5734 00000100:00100000:2.0:1767320232.164866:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5735 00000100:00100000:2.0:1767320232.164870:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985372544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320232.164878:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320232.164913:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985372544:12345-0@lo:38:kworker.0 Request processed in 42us (801us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320232.164919:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5735 00000100:00100000:2.0:1767320232.166059:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5736 00000100:00100000:2.0:1767320232.166067:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985372800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320232.166082:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320232.166122:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985372800:12345-0@lo:38:kworker.0 Request processed in 55us (1965us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320232.166127:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5736 00000100:00080000:3.0:1767320232.180734:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320232.180748:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320232.180754:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985372160:0@lo:38:kworker.0 00000100:00080000:3.0:1767320232.180768:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320232.180772:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320232.180775:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985372288:0@lo:38:kworker.0 00000100:00080000:3.0:1767320232.180785:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320232.180789:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320232.180792:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985372544:0@lo:38:kworker.0 00000100:00080000:3.0:1767320232.180802:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320232.180804:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320232.180807:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985372800:0@lo:38:kworker.0 00010000:00010000:0.0:1767320232.795443:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320233.823600:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320234.843377:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320235.868500:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320236.205467:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320236.205548:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920605696 00000100:00100000:0.0:1767320236.205559:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920605696 00000100:00100000:0.0:1767320236.205639:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5099 00000100:00100000:0.0:1767320236.205650:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920605696:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320236.205730:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920605696:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (272us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320236.205738:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5099 00000100:00080000:1.0:1767320236.206737:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320236.213688:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320236.213811:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920605568 00000100:00100000:3.0:1767320236.213821:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920605568 00000100:00100000:3.0:1767320236.213869:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5737 00000100:00100000:3.0:1767320236.213877:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920605568:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320236.213899:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320236.213971:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920605568:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (291us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320236.213978:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5737 00000100:00100000:0.0:1767320236.763564:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320236.763589:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985373184:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320236.763661:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320236.763773:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373184 00000100:00100000:2.0:1767320236.763781:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373184 00000100:00100000:2.0:1767320236.763860:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2516 00000100:00100000:2.0:1767320236.763871:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985373184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320236.763901:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320236.763967:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985373184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (305us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320236.763975:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2516 00000100:00080000:0.0:1767320236.764185:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320236.764450:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985373184:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320236.764671:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985373312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320236.764705:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320236.764917:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373312 00000100:00100000:3.0:1767320236.764923:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373312 00000100:00080000:0.0:1767320236.765143:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320236.765225:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2517 00000100:00100000:3.0:1767320236.765234:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985373312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320236.765254:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320236.765305:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985373312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (598us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320236.765312:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2517 00000100:00100000:0.0:1767320236.765376:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985373312:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320236.894874:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320237.275766:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320237.275777:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320237.275780:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320237.275783:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320237.275788:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320237.275796:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5480 00000100:00080000:3.0:1767320237.275800:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320237.275806:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320237.275842:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04a80 x1853163985373440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320237.275859:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320237.275862:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320237.275864:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320237.275867:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5480 00000100:00080000:3.0:1767320237.275869:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320237.275873:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320237.275898:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985373568/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320237.275922:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320237.275925:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320237.275927:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320237.275930:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5480 00000100:00080000:3.0:1767320237.275932:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320237.275964:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320237.275978:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985373824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320237.276003:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320237.276006:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320237.276007:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320237.276011:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5480 00000100:00080000:3.0:1767320237.276013:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320237.276017:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320237.276034:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05180 x1853163985374080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320237.276189:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985373952:0@lo:400:kworker.0 00000100:00100000:0.0:1767320237.276251:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320237.276324:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320237.276331:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985373696:0@lo:400:kworker.0 00000100:00100000:0.0:1767320237.276353:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320237.276392:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373952 00000100:00100000:0.0:1767320237.276399:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373952 00000100:00100000:0.0:1767320237.276446:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373696 00000100:00100000:0.0:1767320237.276449:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373696 00000100:00100000:0.0:1767320237.276471:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5100 00000100:00100000:0.0:1767320237.276479:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985373952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320237.276540:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985373952:12345-0@lo:400:kworker.0 Request processed in 60us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320237.276547:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5100 00000100:00100000:0.0:1767320237.276564:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5101 00000100:00100000:0.0:1767320237.276570:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985373696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320237.276597:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985373696:12345-0@lo:400:kworker.0 Request processed in 28us (243us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320237.276603:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5101 00000100:00080000:0.0:1767320237.276619:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320237.276646:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985373696:0@lo:400:kworker.0 00000100:00100000:0.0:1767320237.276655:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985373952:0@lo:400:kworker.0 00000100:00100000:2.0:1767320237.277474:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985373440:0@lo:38:kworker.0 00000100:00100000:2.0:1767320237.277511:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320237.277547:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985373568:0@lo:38:kworker.0 00000100:00100000:2.0:1767320237.277573:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320237.277597:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985373824:0@lo:38:kworker.0 00000100:00100000:2.0:1767320237.277618:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320237.277634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985374080:0@lo:38:kworker.0 00000100:00100000:2.0:1767320237.277654:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320237.277685:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373568 00000100:00100000:2.0:1767320237.277690:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373568 00000100:00100000:2.0:1767320237.277717:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1925 00000100:00100000:2.0:1767320237.277722:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985373568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320237.277741:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320237.277777:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985373568:12345-0@lo:38:kworker.0 Request processed in 53us (202us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320237.277784:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1925 00000100:00100000:2.0:1767320237.277803:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373440 00000100:00100000:2.0:1767320237.277806:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373440 00000100:00100000:2.0:1767320237.277817:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985373824 00000100:00100000:2.0:1767320237.277820:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985373824 00000100:00100000:2.0:1767320237.277826:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985374080 00000100:00100000:2.0:1767320237.277828:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985374080 00000100:00100000:1.0:1767320237.277832:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5738 00000100:00100000:1.0:1767320237.277838:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985373440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320237.277848:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320237.278380:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985373440:12345-0@lo:38:kworker.0 Request processed in 539us (866us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320237.291913:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5739 00000100:00100000:2.0:1767320237.291927:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985373824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320237.291964:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320237.293347:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985373824:12345-0@lo:38:kworker.0 Request processed in 1416us (15723us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320237.293359:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5739 00000100:00100000:2.0:1767320237.293388:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5740 00000100:00100000:2.0:1767320237.293394:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985374080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320237.293410:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320237.293446:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985374080:12345-0@lo:38:kworker.0 Request processed in 51us (15791us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320237.293456:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5740 00000100:00080000:2.0:1767320237.293512:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320237.293518:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320237.293521:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985373440:0@lo:38:kworker.0 00000100:00080000:2.0:1767320237.293535:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320237.293538:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320237.293541:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985373568:0@lo:38:kworker.0 00000100:00080000:2.0:1767320237.293549:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320237.293553:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320237.293556:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985373824:0@lo:38:kworker.0 00000100:00080000:2.0:1767320237.293564:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320237.293567:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320237.293569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985374080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320237.313577:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5738 00010000:00010000:1.0:1767320237.916317:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320238.939458:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320239.963403:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320240.298741:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767320240.298776:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920605824 offset 0 length 520: 4 00000100:00100000:3.0:1767320240.298795:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320240.298922:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606080 00000100:00100000:1.0:1767320240.298932:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606080 00000100:00100000:1.0:1767320240.299007:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920605952 00000100:00100000:1.0:1767320240.299010:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920605952 00000100:00100000:1.0:1767320240.299027:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5102 00000100:00100000:1.0:1767320240.299036:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920606080:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320240.299101:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920606080:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 66us (371us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320240.299109:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5102 00000100:00100000:1.0:1767320240.299124:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5741 00000100:00100000:1.0:1767320240.299129:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920605952:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320240.299200:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320240.299225:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920605952:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (429us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320240.299233:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5741 00000100:00080000:2.0:1767320240.299309:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320240.987634:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320241.371561:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320216/real 1767320216] req@ffff8ce1b351a300 x1853163985368192/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320241 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320241.371593:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b351a300 x1853163985368192/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320241 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320241.371606:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320241.371636:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320241.371640:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985368192:0@lo:250:kworker.0 00000100:00080000:3.0:1767320241.372381:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320241.372387:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320241.372390:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320241.372395:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5465 00000100:00080000:3.0:1767320241.372399:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320241.372405:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320241.372447:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04e00 x1853163985374208/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320241.372473:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320241.372477:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320241.372480:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320241.372483:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5485 00000100:00080000:3.0:1767320241.372486:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320241.372489:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320241.372504:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985374336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320241.372513:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320241.372516:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320241.372518:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320241.372530:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5485 00000100:00080000:3.0:1767320241.372532:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320241.372535:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320241.372547:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05f80 x1853163985374464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320241.372592:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320241.372596:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320241.372598:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320241.372602:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5485 00000100:00080000:3.0:1767320241.372604:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320241.372607:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320241.372618:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985374720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320241.372634:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320241.372637:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320241.372639:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320241.372642:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5485 00000100:00080000:3.0:1767320241.372644:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320241.372647:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320241.372658:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04a80 x1853163985374976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320241.372705:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985375232:0@lo:400:kworker.0 00000100:00100000:3.0:1767320241.372763:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320241.372820:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320241.372826:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985375104:0@lo:400:kworker.0 00000100:00100000:3.0:1767320241.372851:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320241.372892:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985375232 00000100:00100000:3.0:1767320241.372899:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985375232 00000100:00100000:3.0:1767320241.372945:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985375104 00000100:00100000:3.0:1767320241.372948:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985375104 00000100:00100000:3.0:1767320241.372970:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3321 00000100:00100000:3.0:1767320241.372978:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985375232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320241.373026:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985375232:12345-0@lo:400:kworker.0 Request processed in 48us (262us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320241.373033:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3321 00000100:00100000:3.0:1767320241.373048:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3322 00000100:00100000:3.0:1767320241.373054:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985375104:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320241.373080:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985375104:12345-0@lo:400:kworker.0 Request processed in 26us (227us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320241.373085:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3322 00000100:00100000:3.0:1767320241.373123:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985375104:0@lo:400:kworker.0 00000100:00100000:3.0:1767320241.373133:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985375232:0@lo:400:kworker.0 00000100:00100000:2.0:1767320241.373373:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985374848:0@lo:400:kworker.0 00000100:00100000:2.0:1767320241.373403:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320241.373435:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320241.373440:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985374592:0@lo:400:kworker.0 00000100:00100000:2.0:1767320241.373459:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320241.373490:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985374848 00000100:00100000:2.0:1767320241.373494:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985374848 00000100:00100000:2.0:1767320241.373521:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985374592 00000100:00100000:2.0:1767320241.373532:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985374592 00000100:00100000:2.0:1767320241.373556:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5103 00000100:00100000:2.0:1767320241.373562:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985374848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320241.373594:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985374848:12345-0@lo:400:kworker.0 Request processed in 33us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320241.373600:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5103 00000100:00100000:2.0:1767320241.373613:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5104 00000100:00100000:2.0:1767320241.373620:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985374592:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320241.373645:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985374592:12345-0@lo:400:kworker.0 Request processed in 26us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320241.373650:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5104 00000100:00080000:2.0:1767320241.373672:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320241.373676:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320241.373678:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320241.373702:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985374592:0@lo:400:kworker.0 00000100:00100000:2.0:1767320241.373711:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985374848:0@lo:400:kworker.0 00000100:00100000:1.0:1767320241.374133:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985374208:0@lo:250:kworker.0 00000400:00000100:1.0:1767320241.374193:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985374208 offset 0 length 520: 4 00000100:00100000:1.0:1767320241.374216:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985374336:0@lo:38:kworker.0 00000100:00100000:1.0:1767320241.374240:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320241.374265:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985374464:0@lo:38:kworker.0 00000100:00100000:1.0:1767320241.374295:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320241.374311:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985374720:0@lo:38:kworker.0 00000100:00100000:1.0:1767320241.374331:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320241.374346:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985374976:0@lo:38:kworker.0 00000100:00100000:1.0:1767320241.374363:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320241.374743:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985374336 00000100:00100000:3.0:1767320241.374751:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985374336 00000100:00100000:3.0:1767320241.374785:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985374720 00000100:00100000:3.0:1767320241.374788:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985374720 00000100:00100000:3.0:1767320241.374808:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985374976 00000100:00100000:3.0:1767320241.374810:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985374976 00000100:00100000:3.0:1767320241.374821:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5742 00000100:00100000:3.0:1767320241.374828:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985374336:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320241.374845:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320241.374914:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985374336:12345-0@lo:38:kworker.0 Request processed in 85us (673us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320241.374922:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5742 00000100:00100000:3.0:1767320241.374935:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5743 00000100:00100000:3.0:1767320241.374938:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985374720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320241.374947:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320241.374968:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985374720:12345-0@lo:38:kworker.0 Request processed in 28us (636us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320241.374974:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5743 00000100:00100000:3.0:1767320241.374983:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5744 00000100:00100000:3.0:1767320241.374986:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985374976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320241.374994:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320241.375013:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985374976:12345-0@lo:38:kworker.0 Request processed in 25us (649us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320241.375018:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5744 00000100:00100000:2.0:1767320241.375537:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985374464 00000100:00100000:2.0:1767320241.375542:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985374464 00000100:00100000:2.0:1767320241.375571:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1926 00000100:00100000:2.0:1767320241.375576:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985374464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320241.375586:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320241.375617:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985374464:12345-0@lo:38:kworker.0 Request processed in 41us (1322us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320241.375622:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1926 00000100:00080000:1.0:1767320241.376451:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320241.376458:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320241.376462:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985374336:0@lo:38:kworker.0 00000100:00080000:1.0:1767320241.376472:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320241.376475:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320241.376478:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985374464:0@lo:38:kworker.0 00000100:00080000:1.0:1767320241.376487:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320241.376491:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320241.376493:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985374720:0@lo:38:kworker.0 00000100:00080000:1.0:1767320241.376503:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320241.376505:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320241.376508:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985374976:0@lo:38:kworker.0 00000100:00100000:0.0:1767320241.883662:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985375360:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320241.883732:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320241.884024:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320241.884034:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985375488:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320241.884062:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320241.884134:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985375360 00000100:00100000:0.0:1767320241.884142:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985375360 00000100:00100000:0.0:1767320241.884205:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985375488 00000100:00100000:0.0:1767320241.884209:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985375488 00000100:00100000:0.0:1767320241.884227:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2518 00000100:00100000:0.0:1767320241.884235:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985375360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320241.884264:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320241.884314:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985375360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (582us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320241.884321:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2518 00000100:00100000:0.0:1767320241.884337:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2519 00000100:00100000:0.0:1767320241.884344:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985375488:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320241.884359:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320241.884383:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985375488:12345-0@lo:13:osp-pre-0-1.0 Request processed in 40us (319us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320241.884389:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2519 00000100:00100000:2.0:1767320241.884654:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985375488:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320241.884669:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985375360:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320242.011714:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320243.035389:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320244.060960:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320245.083621:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320245.418618:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320245.418702:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320245.418912:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606336 00000100:00100000:3.0:1767320245.418923:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606336 00000100:00100000:3.0:1767320245.418997:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606208 00000100:00100000:3.0:1767320245.419000:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606208 00000100:00100000:3.0:1767320245.419027:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5105 00000100:00100000:3.0:1767320245.419038:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920606336:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320245.419063:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5745 00000100:00100000:0.0:1767320245.419078:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920606208:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320245.419110:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320245.419140:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920606336:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (520us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320245.419150:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5105 00000100:00100000:0.0:1767320245.419311:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920606208:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 231us (606us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320245.419324:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5745 00000100:00080000:2.0:1767320245.419337:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320246.107404:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320246.491781:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320246.491793:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320246.491795:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320246.491799:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320246.491803:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320246.491812:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5489 00000100:00080000:3.0:1767320246.491817:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320246.491824:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320246.491866:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985375616/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320246.491892:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320246.491896:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320246.491898:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320246.491901:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5489 00000100:00080000:3.0:1767320246.491903:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320246.491907:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320246.491920:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05f80 x1853163985375744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320246.491959:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320246.491962:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320246.491964:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320246.491967:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5489 00000100:00080000:3.0:1767320246.491970:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320246.491972:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320246.491982:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985376000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320246.491995:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320246.491998:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320246.492000:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320246.492003:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5489 00000100:00080000:3.0:1767320246.492006:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320246.492009:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320246.492017:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06d80 x1853163985376256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320246.492106:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985376128:0@lo:400:kworker.0 00000100:00100000:0.0:1767320246.492205:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320246.492260:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320246.492265:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985375872:0@lo:400:kworker.0 00000100:00100000:0.0:1767320246.492285:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320246.492332:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376128 00000100:00100000:0.0:1767320246.492338:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376128 00000100:00100000:0.0:1767320246.492378:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985375872 00000100:00100000:0.0:1767320246.492390:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985375872 00000100:00100000:0.0:1767320246.492409:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5106 00000100:00100000:0.0:1767320246.492417:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985376128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320246.492462:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985376128:12345-0@lo:400:kworker.0 Request processed in 46us (257us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320246.492469:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5106 00000100:00100000:0.0:1767320246.492498:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5107 00000100:00100000:0.0:1767320246.492504:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985375872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320246.492527:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985375872:12345-0@lo:400:kworker.0 Request processed in 24us (242us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320246.492533:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5107 00000100:00100000:0.0:1767320246.492568:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985375872:0@lo:400:kworker.0 00000100:00100000:0.0:1767320246.492575:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985376128:0@lo:400:kworker.0 00000100:00080000:2.0:1767320246.493302:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320246.493330:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985376384:0@lo:400:kworker.0 00000100:00100000:2.0:1767320246.493371:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320246.493410:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320246.493416:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985376512:0@lo:400:kworker.0 00000100:00100000:2.0:1767320246.493434:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320246.493460:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376384 00000100:00100000:2.0:1767320246.493464:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376384 00000100:00100000:2.0:1767320246.493636:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376512 00000100:00100000:2.0:1767320246.493640:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376512 00000100:00100000:2.0:1767320246.493661:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3323 00000100:00100000:2.0:1767320246.493667:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985376384:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320246.493709:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985376384:12345-0@lo:400:kworker.0 Request processed in 41us (336us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320246.493715:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3323 00000100:00100000:2.0:1767320246.493728:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3324 00000100:00100000:2.0:1767320246.493734:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985376512:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320246.493758:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985376512:12345-0@lo:400:kworker.0 Request processed in 24us (322us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320246.493775:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3324 00000100:00080000:2.0:1767320246.493793:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320246.493796:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320246.493816:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985376512:0@lo:400:kworker.0 00000100:00100000:2.0:1767320246.493824:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985376384:0@lo:400:kworker.0 00000100:00100000:1.0:1767320246.494647:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985375616:0@lo:38:kworker.0 00000100:00100000:1.0:1767320246.494697:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320246.494739:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985375744:0@lo:38:kworker.0 00000100:00100000:1.0:1767320246.494784:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320246.494805:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985376000:0@lo:38:kworker.0 00000100:00100000:1.0:1767320246.494825:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320246.494851:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985376256:0@lo:38:kworker.0 00000100:00100000:1.0:1767320246.494871:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320246.494909:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985375616 00000100:00100000:1.0:1767320246.494914:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985375616 00000100:00100000:1.0:1767320246.494941:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376000 00000100:00100000:1.0:1767320246.494944:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376000 00000100:00100000:1.0:1767320246.494957:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376256 00000100:00100000:1.0:1767320246.494959:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376256 00000100:00100000:1.0:1767320246.494973:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5746 00000100:00100000:1.0:1767320246.494978:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985375616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320246.494997:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320246.495035:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985375616:12345-0@lo:38:kworker.0 Request processed in 55us (336us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320246.495042:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5746 00000100:00100000:1.0:1767320246.495056:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5747 00000100:00100000:1.0:1767320246.495061:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985376000:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320246.495069:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320246.495442:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985375744 00000100:00100000:0.0:1767320246.495447:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985375744 00000100:00100000:0.0:1767320246.495466:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1927 00000100:00100000:0.0:1767320246.495471:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985375744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320246.495481:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320246.495519:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985375744:12345-0@lo:38:kworker.0 Request processed in 47us (734us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320246.495525:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1927 00000100:00100000:0.0:1767320246.495568:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5748 00000100:00100000:0.0:1767320246.495573:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985376256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320246.495581:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320246.495602:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985376256:12345-0@lo:38:kworker.0 Request processed in 28us (731us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320246.495607:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5748 00000100:00080000:0.0:1767320246.495648:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320246.495653:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320246.495657:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985375616:0@lo:38:kworker.0 00000100:00080000:0.0:1767320246.495665:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320246.495669:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320246.495671:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985375744:0@lo:38:kworker.0 00000100:00080000:0.0:1767320246.495679:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320246.495682:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320246.495684:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985376256:0@lo:38:kworker.0 00000100:00100000:1.0:1767320246.496017:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985376000:12345-0@lo:38:kworker.0 Request processed in 953us (1189us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320246.496024:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5747 00000100:00080000:0.0:1767320246.498062:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320246.498073:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320246.498090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985376000:0@lo:38:kworker.0 00000100:00100000:1.0:1767320247.003652:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320247.003678:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985376640:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320247.003746:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320247.003951:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985376768:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320247.003978:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320247.004062:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376640 00000100:00100000:3.0:1767320247.004070:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376640 00000100:00100000:3.0:1767320247.004126:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376768 00000100:00100000:3.0:1767320247.004130:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376768 00000100:00100000:3.0:1767320247.004150:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2520 00000100:00100000:3.0:1767320247.004158:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985376640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320247.004218:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320247.004267:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985376640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (515us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320247.004275:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2520 00000100:00100000:3.0:1767320247.004293:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2521 00000100:00100000:3.0:1767320247.004299:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985376768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320247.004314:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320247.004344:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985376768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (365us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320247.004351:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2521 00000100:00100000:1.0:1767320247.004534:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985376768:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320247.004549:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985376640:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320247.131631:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320248.155558:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320249.179826:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320250.203788:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320250.540246:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320250.540338:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320250.540362:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606592 00000100:00100000:3.0:1767320250.540375:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606592 00000100:00100000:3.0:1767320250.540450:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606464 00000100:00100000:3.0:1767320250.540453:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606464 00000100:00100000:3.0:1767320250.540474:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5108 00000100:00100000:3.0:1767320250.540485:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920606592:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320250.540496:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5749 00000100:00100000:0.0:1767320250.540505:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920606464:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320250.540527:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320250.540567:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920606592:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (328us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320250.540577:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5108 00000100:00100000:0.0:1767320250.540579:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920606464:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 60us (228us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320250.540590:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5749 00000100:00080000:2.0:1767320250.540830:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320251.231227:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320251.611889:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320251.611908:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320251.611911:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320251.611915:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320251.611920:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320251.611929:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5494 00000100:00080000:2.0:1767320251.611934:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320251.611941:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320251.612412:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3b80 x1853163985376896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320251.612439:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320251.612442:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320251.612444:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320251.612448:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5494 00000100:00080000:2.0:1767320251.612451:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320251.612455:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320251.612468:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1c00 x1853163985377024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320251.612547:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320251.612593:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985376896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320251.612661:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320251.612712:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985376896 00000100:00100000:3.0:1767320251.612721:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985376896 00000100:00100000:1.0:1767320251.612731:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985377024:0@lo:38:kworker.0 00000100:00100000:3.0:1767320251.612764:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5750 00000100:00100000:3.0:1767320251.612772:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985376896:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320251.612775:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767320251.612797:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320251.612841:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377024 00000100:00100000:1.0:1767320251.612846:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377024 00000100:00100000:3.0:1767320251.612849:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985376896:12345-0@lo:38:kworker.0 Request processed in 76us (187us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320251.612859:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5750 00000100:00100000:1.0:1767320251.612875:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1928 00000100:00100000:1.0:1767320251.612882:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985377024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320251.612894:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320251.612934:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985377024:12345-0@lo:38:kworker.0 Request processed in 51us (160us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320251.612940:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1928 00000100:00080000:1.0:1767320251.612984:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320251.612989:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320251.612992:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985376896:0@lo:38:kworker.0 00000100:00080000:1.0:1767320251.613002:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320251.613005:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320251.613007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985377024:0@lo:38:kworker.0 00000100:00080000:2.0:1767320251.613221:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320251.613225:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320251.613227:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320251.613242:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5494 00000100:00080000:2.0:1767320251.613245:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320251.613249:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320251.613264:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1180 x1853163985377280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320251.613327:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985377280:0@lo:38:kworker.0 00000100:00100000:1.0:1767320251.613354:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320251.613510:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377280 00000100:00100000:3.0:1767320251.613525:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377280 00000100:00100000:3.0:1767320251.613555:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5751 00000100:00100000:3.0:1767320251.613560:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985377280:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320251.613570:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320251.613601:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985377280:12345-0@lo:38:kworker.0 Request processed in 39us (246us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320251.613607:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5751 00000100:00080000:1.0:1767320251.613737:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320251.613742:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320251.613745:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985377280:0@lo:38:kworker.0 00000100:00100000:0.0:1767320251.618600:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985377408:0@lo:400:kworker.0 00000100:00100000:0.0:1767320251.618688:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320251.618730:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985377152:0@lo:400:kworker.0 00000100:00100000:1.0:1767320251.618773:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377408 00000100:00100000:1.0:1767320251.618782:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377408 00000100:00100000:1.0:1767320251.618852:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5109 00000100:00100000:1.0:1767320251.618862:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985377408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320251.618917:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985377408:12345-0@lo:400:kworker.0 Request processed in 56us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320251.618923:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5109 00000100:00080000:3.0:1767320251.619005:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320251.619116:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320251.619191:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985377408:0@lo:400:kworker.0 00000100:00080000:2.0:1767320251.619474:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320251.619479:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320251.619483:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320251.619498:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5494 00000100:00080000:2.0:1767320251.619502:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320251.619506:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320251.619534:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0000 x1853163985377536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320251.619560:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377152 00000100:00100000:3.0:1767320251.619566:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377152 00000100:00100000:2.0:1767320251.619600:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985377792:0@lo:400:kworker.0 00000100:00100000:3.0:1767320251.619611:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5110 00000100:00100000:3.0:1767320251.619620:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985377152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320251.619638:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320251.619667:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320251.619668:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985377152:12345-0@lo:400:kworker.0 Request processed in 49us (551us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320251.619685:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985377664:0@lo:400:kworker.0 00000100:00100000:3.0:1767320251.619690:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5110 00000100:00080000:3.0:1767320251.619713:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320251.619714:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320251.619810:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985377536:0@lo:38:kworker.0 00000100:00100000:1.0:1767320251.619840:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320251.620594:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985377152:0@lo:400:kworker.0 00000100:00100000:2.0:1767320251.620740:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377792 00000100:00100000:2.0:1767320251.620756:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377792 00000100:00100000:2.0:1767320251.620800:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377664 00000100:00100000:2.0:1767320251.620804:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377664 00000100:00100000:2.0:1767320251.620838:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3325 00000100:00100000:2.0:1767320251.620847:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985377792:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320251.620908:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985377792:12345-0@lo:400:kworker.0 Request processed in 53us (1260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320251.620916:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3325 00000100:00100000:2.0:1767320251.620931:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3326 00000100:00100000:2.0:1767320251.620937:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985377664:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320251.620978:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985377664:12345-0@lo:400:kworker.0 Request processed in 42us (1265us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320251.620984:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3326 00000100:00080000:2.0:1767320251.621005:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320251.621009:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320251.621027:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985377664:0@lo:400:kworker.0 00000100:00100000:2.0:1767320251.621037:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985377792:0@lo:400:kworker.0 00000100:00100000:3.0:1767320251.621642:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377536 00000100:00100000:3.0:1767320251.621649:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377536 00000100:00100000:3.0:1767320251.621696:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5752 00000100:00100000:3.0:1767320251.621702:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985377536:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320251.621721:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320251.621765:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985377536:12345-0@lo:38:kworker.0 Request processed in 62us (1924us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320251.621772:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5752 00000100:00080000:1.0:1767320251.621872:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320251.621878:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320251.621882:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985377536:0@lo:38:kworker.0 00000100:00100000:1.0:1767320252.123724:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985377920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320252.123799:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320252.126362:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985377920 00000100:00100000:0.0:1767320252.126377:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985377920 00000100:00100000:0.0:1767320252.126450:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2522 00000100:00100000:0.0:1767320252.126460:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985377920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320252.126493:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320252.126576:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985377920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 118us (2776us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320252.126583:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2522 00000100:00100000:2.0:1767320252.126687:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985377920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320252.126921:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320252.126934:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985378048:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320252.126973:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320252.127048:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378048 00000100:00100000:0.0:1767320252.127053:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378048 00000100:00100000:0.0:1767320252.127086:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2523 00000100:00100000:0.0:1767320252.127092:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985378048:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320252.127109:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320252.127150:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985378048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (176us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320252.127157:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2523 00000100:00100000:3.0:1767320252.127318:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985378048:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320252.251424:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320253.275635:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320254.300941:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320255.323518:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320255.667423:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320255.667473:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320255.667718:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606720 00000100:00100000:2.0:1767320255.667729:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606720 00000100:00100000:2.0:1767320255.667773:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606848 00000100:00100000:2.0:1767320255.667777:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606848 00000100:00100000:2.0:1767320255.667828:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5111 00000100:00100000:2.0:1767320255.667839:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920606848:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320255.667919:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920606848:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (445us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320255.667927:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5111 00000100:00100000:2.0:1767320255.667945:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5753 00000100:00100000:2.0:1767320255.667950:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920606720:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320255.667969:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320255.668006:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920606720:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (580us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320255.668012:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5753 00000100:00080000:1.0:1767320255.668311:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320256.350958:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320256.731244:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320256.731254:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320256.731257:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320256.731261:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320256.731265:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320256.731274:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5499 00000100:00080000:3.0:1767320256.731278:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320256.731284:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320256.731321:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985378176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320256.731350:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320256.731353:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320256.731355:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320256.731359:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5499 00000100:00080000:3.0:1767320256.731361:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320256.731364:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320256.731376:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05880 x1853163985378304/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320256.731419:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320256.731423:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320256.731425:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320256.731429:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5499 00000100:00080000:3.0:1767320256.731431:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320256.731435:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320256.731445:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05180 x1853163985378560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320256.731460:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320256.731463:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320256.731465:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320256.731469:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5499 00000100:00080000:3.0:1767320256.731471:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320256.731474:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320256.731496:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985378816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320256.731563:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985378944:0@lo:400:kworker.0 00000100:00100000:3.0:1767320256.731652:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320256.731709:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320256.731716:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985379072:0@lo:400:kworker.0 00000100:00100000:3.0:1767320256.731737:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320256.731821:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985378688:0@lo:400:kworker.0 00000100:00100000:1.0:1767320256.731851:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320256.731886:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320256.731891:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985378432:0@lo:400:kworker.0 00000100:00100000:1.0:1767320256.731911:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320256.731946:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378688 00000100:00100000:1.0:1767320256.731953:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378688 00000100:00100000:1.0:1767320256.732000:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378432 00000100:00100000:1.0:1767320256.732003:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378432 00000100:00100000:1.0:1767320256.732024:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5112 00000100:00100000:1.0:1767320256.732032:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985378688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320256.732074:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378944 00000100:00100000:2.0:1767320256.732078:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378944 00000100:00100000:1.0:1767320256.732079:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985378688:12345-0@lo:400:kworker.0 Request processed in 48us (227us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320256.732086:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5112 00000100:00100000:1.0:1767320256.732100:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5113 00000100:00100000:2.0:1767320256.732103:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379072 00000100:00100000:2.0:1767320256.732105:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379072 00000100:00100000:1.0:1767320256.732106:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985378432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320256.732123:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3327 00000100:00100000:2.0:1767320256.732129:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985378944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320256.732133:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985378432:12345-0@lo:400:kworker.0 Request processed in 28us (222us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320256.732138:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5113 00000100:00080000:1.0:1767320256.732153:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320256.732158:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320256.732160:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320256.732160:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985378944:12345-0@lo:400:kworker.0 Request processed in 33us (509us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320256.732185:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3327 00000100:00100000:2.0:1767320256.732197:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3328 00000100:00100000:2.0:1767320256.732203:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985379072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320256.732209:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985378432:0@lo:400:kworker.0 00000100:00100000:1.0:1767320256.732219:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985378688:0@lo:400:kworker.0 00000100:00100000:2.0:1767320256.732227:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985379072:12345-0@lo:400:kworker.0 Request processed in 26us (490us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320256.732232:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3328 00000100:00100000:1.0:1767320256.732457:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985379072:0@lo:400:kworker.0 00000100:00100000:1.0:1767320256.732470:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985378944:0@lo:400:kworker.0 00000100:00100000:1.0:1767320256.739010:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985378176:0@lo:38:kworker.0 00000100:00100000:1.0:1767320256.739087:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320256.739133:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378176 00000100:00100000:1.0:1767320256.739138:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985378304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320256.739140:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378176 00000100:00100000:1.0:1767320256.739161:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320256.739198:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5754 00000100:00100000:1.0:1767320256.739202:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985378560:0@lo:38:kworker.0 00000100:00100000:2.0:1767320256.739205:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985378176:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320256.739222:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767320256.739225:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320256.739238:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985378816:0@lo:38:kworker.0 00000100:00100000:1.0:1767320256.739255:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320256.739288:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378304 00000100:00100000:1.0:1767320256.739292:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378304 00000100:00100000:1.0:1767320256.739312:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1929 00000100:00100000:1.0:1767320256.739317:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985378304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320256.739327:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320256.739358:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985378304:12345-0@lo:38:kworker.0 Request processed in 39us (195us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320256.739364:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1929 00000100:00100000:1.0:1767320256.739386:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378560 00000100:00100000:1.0:1767320256.739389:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378560 00000100:00100000:1.0:1767320256.739410:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985378816 00000100:00100000:1.0:1767320256.739412:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985378816 00000100:00100000:1.0:1767320256.739426:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5755 00000100:00100000:1.0:1767320256.739430:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985378560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320256.739439:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320256.739443:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985378176:12345-0@lo:38:kworker.0 Request processed in 238us (356us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320256.739449:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5754 00000100:00100000:2.0:1767320256.739462:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5756 00000100:00100000:1.0:1767320256.739464:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985378560:12345-0@lo:38:kworker.0 Request processed in 33us (242us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320256.739466:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985378816:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320256.739471:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5755 00010000:00020000:2.0:1767320256.739475:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320256.739496:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985378816:12345-0@lo:38:kworker.0 Request processed in 28us (239us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320256.739501:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5756 00000100:00080000:1.0:1767320256.739517:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320256.739522:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320256.739526:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985378176:0@lo:38:kworker.0 00000100:00080000:1.0:1767320256.739537:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320256.739541:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320256.739543:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985378304:0@lo:38:kworker.0 00000100:00080000:2.0:1767320256.739600:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320256.739605:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320256.739607:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985378560:0@lo:38:kworker.0 00000100:00080000:2.0:1767320256.739614:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320256.739617:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320256.739620:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985378816:0@lo:38:kworker.0 00000100:00100000:1.0:1767320257.243419:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320257.243443:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985379200:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320257.243519:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320257.243704:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379200 00000100:00100000:3.0:1767320257.243713:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379200 00000100:00100000:3.0:1767320257.243769:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2524 00000100:00100000:3.0:1767320257.243779:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985379200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320257.243809:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320257.243859:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985379200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (338us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320257.243867:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2524 00000100:00100000:3.0:1767320257.243916:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985379200:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320257.252251:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320257.252277:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985379328:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320257.252367:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320257.252663:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379328 00000100:00100000:0.0:1767320257.252672:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379328 00000100:00100000:0.0:1767320257.252732:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2525 00000100:00100000:0.0:1767320257.252743:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985379328:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320257.252775:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320257.252831:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985379328:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (464us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320257.252839:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2525 00000100:00100000:2.0:1767320257.252968:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985379328:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320257.377235:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320258.396781:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320259.420876:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320260.443389:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320260.779240:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320260.779276:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320260.779344:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920607104 00000100:00100000:1.0:1767320260.779345:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920606976 00000100:00100000:0.0:1767320260.779354:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920607104 00000100:00100000:1.0:1767320260.779354:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920606976 00000100:00100000:0.0:1767320260.779412:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5114 00000100:00100000:0.0:1767320260.779420:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920607104:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320260.779479:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5757 00000100:00100000:1.0:1767320260.779487:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920606976:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:0.0:1767320260.779491:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920607104:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 70us (258us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320260.779499:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5114 00010000:00020000:1.0:1767320260.779528:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320260.779561:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920606976:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 73us (282us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320260.779570:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5757 00000100:00080000:2.0:1767320260.779653:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320261.467401:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320261.851561:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320261.851572:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320261.851575:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320261.851579:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320261.851584:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320261.851591:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5504 00000100:00080000:1.0:1767320261.851596:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320261.851602:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320261.851648:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357aa00 x1853163985379456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320261.851678:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320261.851682:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320261.851684:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320261.851687:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5504 00000100:00080000:1.0:1767320261.851690:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320261.851693:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320261.851707:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985379584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320261.851733:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320261.851737:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320261.851738:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320261.851741:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5504 00000100:00080000:1.0:1767320261.851744:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320261.851746:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320261.851755:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05880 x1853163985379840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320261.851768:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320261.851771:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320261.851772:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320261.851776:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5504 00000100:00080000:1.0:1767320261.851778:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320261.851781:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320261.851789:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985380096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320261.852059:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985379456:0@lo:38:kworker.0 00000100:00100000:0.0:1767320261.852131:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320261.852199:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985379584:0@lo:38:kworker.0 00000100:00100000:0.0:1767320261.852226:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320261.852252:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985379840:0@lo:38:kworker.0 00000100:00100000:3.0:1767320261.852270:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379584 00000100:00100000:0.0:1767320261.852271:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320261.852280:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379584 00000100:00100000:0.0:1767320261.852288:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985380096:0@lo:38:kworker.0 00000100:00100000:0.0:1767320261.852307:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320261.852320:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1930 00000100:00100000:3.0:1767320261.852330:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985379584:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320261.852341:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379456 00000100:00100000:0.0:1767320261.852345:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379456 00010000:00020000:3.0:1767320261.852350:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320261.852365:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379840 00000100:00100000:0.0:1767320261.852369:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379840 00000100:00100000:0.0:1767320261.852382:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380096 00000100:00100000:0.0:1767320261.852384:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380096 00000100:00100000:3.0:1767320261.852400:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985379584:12345-0@lo:38:kworker.0 Request processed in 70us (173us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320261.852405:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5758 00000100:00100000:3.0:1767320261.852414:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1930 00000100:00100000:0.0:1767320261.852415:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985379456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320261.852428:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320261.852458:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985379456:12345-0@lo:38:kworker.0 Request processed in 44us (328us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320261.852461:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5759 00000100:00100000:0.0:1767320261.852471:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5758 00000100:00100000:3.0:1767320261.852473:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985379840:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320261.852495:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320261.852501:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5760 00000100:00100000:0.0:1767320261.852516:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985380096:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320261.852525:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985379840:12345-0@lo:38:kworker.0 Request processed in 53us (252us total) trans 0 rc -19/-19 00010000:00020000:0.0:1767320261.852527:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320261.852536:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5759 00000100:00100000:0.0:1767320261.852561:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985380096:12345-0@lo:38:kworker.0 Request processed in 44us (252us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320261.852569:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5760 00000100:00100000:3.0:1767320261.852591:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985380352:0@lo:400:kworker.0 00000100:00080000:0.0:1767320261.852605:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320261.852611:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320261.852616:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985379456:0@lo:38:kworker.0 00000100:00100000:3.0:1767320261.852624:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320261.852645:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320261.852649:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985380224:0@lo:400:kworker.0 00000100:00100000:3.0:1767320261.852667:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320261.852690:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985379712:0@lo:400:kworker.0 00000100:00100000:3.0:1767320261.852711:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320261.852726:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320261.852731:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985379968:0@lo:400:kworker.0 00000100:00100000:3.0:1767320261.852749:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320261.852770:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379712 00000100:00100000:3.0:1767320261.852774:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379712 00000100:00100000:3.0:1767320261.852807:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985379968 00000100:00100000:3.0:1767320261.852810:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985379968 00000100:00100000:3.0:1767320261.852832:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5115 00000100:00100000:3.0:1767320261.852838:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985379712:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320261.852877:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985379712:12345-0@lo:400:kworker.0 Request processed in 39us (165us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320261.852884:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5115 00000100:00100000:3.0:1767320261.852896:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5116 00000100:00100000:3.0:1767320261.852903:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985379968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320261.852909:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380352 00000100:00100000:1.0:1767320261.852914:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380352 00000100:00100000:3.0:1767320261.852934:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985379968:12345-0@lo:400:kworker.0 Request processed in 31us (182us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320261.852940:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5116 00000100:00100000:1.0:1767320261.852946:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380224 00000100:00100000:1.0:1767320261.852950:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380224 00000100:00100000:1.0:1767320261.852973:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3329 00000100:00100000:3.0:1767320261.852979:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985379968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320261.852983:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985380352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320261.852996:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985379712:0@lo:400:kworker.0 00000100:00100000:1.0:1767320261.853033:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985380352:12345-0@lo:400:kworker.0 Request processed in 53us (410us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320261.853127:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320261.853132:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320261.853134:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320261.853244:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320261.853250:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320261.853253:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985379584:0@lo:38:kworker.0 00000100:00080000:0.0:1767320261.853266:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320261.853269:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320261.853272:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985379840:0@lo:38:kworker.0 00000100:00080000:0.0:1767320261.853279:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320261.853282:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320261.853285:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985380096:0@lo:38:kworker.0 00000100:00100000:3.0:1767320261.853412:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3330 00000100:00100000:3.0:1767320261.853422:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985380224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320261.853453:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985380224:12345-0@lo:400:kworker.0 Request processed in 34us (785us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320261.853625:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3329 00000100:00100000:2.0:1767320261.870634:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985380224:0@lo:400:kworker.0 00000100:00100000:2.0:1767320261.870688:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985380352:0@lo:400:kworker.0 00000100:00100000:3.0:1767320261.882518:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3330 00000100:00100000:0.0:1767320262.363828:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985380480:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320262.363896:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320262.365293:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320262.365310:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985380608:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320262.365354:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320262.365392:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380480 00000100:00100000:0.0:1767320262.365399:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380480 00000100:00100000:0.0:1767320262.365442:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380608 00000100:00100000:0.0:1767320262.365445:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380608 00000100:00100000:0.0:1767320262.365465:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2526 00000100:00100000:0.0:1767320262.365483:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985380480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320262.365512:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320262.365635:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2527 00000100:00100000:1.0:1767320262.365648:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985380608:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320262.365650:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985380480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 176us (1751us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320262.365662:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2526 00002000:00000024:1.0:1767320262.365670:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320262.365712:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985380608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 69us (358us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320262.365720:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985380480:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320262.365721:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2527 00000100:00100000:0.0:1767320262.365753:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985380608:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320262.491423:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320263.515413:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320264.544969:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320265.564270:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320265.899030:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320265.899065:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767320265.899749:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920607488 offset 0 length 520: 4 00000100:00100000:0.0:1767320265.899848:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920607360 00000100:00100000:0.0:1767320265.899858:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920607360 00000100:00100000:0.0:1767320265.899998:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920607232 00000100:00100000:0.0:1767320265.900003:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920607232 00000100:00100000:0.0:1767320265.900025:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5117 00000100:00100000:0.0:1767320265.900035:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920607360:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320265.900111:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920607360:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (1092us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320265.900119:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5117 00000100:00100000:0.0:1767320265.900137:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5761 00000100:00100000:0.0:1767320265.900142:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920607232:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320265.900180:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320265.900209:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920607232:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 65us (1142us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320265.900216:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5761 00000100:00080000:3.0:1767320265.900524:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320266.588666:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320266.971282:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320266.971293:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320266.971296:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320266.971300:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320266.971305:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320266.971314:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5510 00000100:00080000:1.0:1767320266.971319:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320266.971325:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320266.971372:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c380 x1853163985380736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320266.971407:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320266.971411:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320266.971413:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320266.971416:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5510 00000100:00080000:1.0:1767320266.971418:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320266.971422:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320266.971436:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163985380864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320266.971484:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320266.971488:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320266.971490:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320266.971493:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5510 00000100:00080000:1.0:1767320266.971496:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320266.971498:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320266.971516:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ce00 x1853163985381120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320266.971533:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320266.971537:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320266.971539:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320266.971542:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5510 00000100:00080000:1.0:1767320266.971544:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320266.971547:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320266.971563:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c700 x1853163985381376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320266.971630:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985381504:0@lo:400:kworker.0 00000100:00100000:3.0:1767320266.971695:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985381248:0@lo:400:kworker.0 00000100:00100000:1.0:1767320266.971701:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320266.971747:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320266.971771:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320266.971780:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985381632:0@lo:400:kworker.0 00000100:00100000:3.0:1767320266.971785:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320266.971795:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985380992:0@lo:400:kworker.0 00000100:00100000:1.0:1767320266.971807:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320266.971839:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320266.971857:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985381504 00000100:00100000:1.0:1767320266.971866:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985381504 00000100:00100000:3.0:1767320266.971892:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985381248 00000100:00100000:3.0:1767320266.971898:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985381248 00000100:00100000:1.0:1767320266.971925:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985381632 00000100:00100000:1.0:1767320266.971930:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985381632 00000100:00100000:3.0:1767320266.971949:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380992 00000100:00100000:3.0:1767320266.971953:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380992 00000100:00100000:1.0:1767320266.971970:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3331 00000100:00100000:1.0:1767320266.971981:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985381504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320266.971987:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5118 00000100:00100000:3.0:1767320266.971998:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985381248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320266.972044:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985381248:12345-0@lo:400:kworker.0 Request processed in 48us (296us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320266.972052:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5118 00000100:00100000:3.0:1767320266.972069:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5119 00000100:00100000:3.0:1767320266.972075:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985380992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320266.972149:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3332 00000100:00100000:2.0:1767320266.972159:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985381632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320266.972218:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985381632:12345-0@lo:400:kworker.0 Request processed in 62us (412us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320266.972224:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3332 00000100:00100000:2.0:1767320266.972265:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985381632:0@lo:400:kworker.0 00000100:00080000:2.0:1767320266.972305:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320266.972310:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320266.972312:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320266.972331:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985381248:0@lo:400:kworker.0 00000100:00100000:1.0:1767320266.972432:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985381504:12345-0@lo:400:kworker.0 Request processed in 453us (733us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320266.972439:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3331 00000100:00100000:3.0:1767320266.972535:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985380992:12345-0@lo:400:kworker.0 Request processed in 459us (695us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320266.972542:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5119 00000100:00100000:0.0:1767320266.972661:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320241/real 1767320241] req@ffff8ce188f04e00 x1853163985374208/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320266 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320266.972689:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188f04e00 x1853163985374208/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320266 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320266.972706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985380736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320266.972737:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985380992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320266.972744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320266.972775:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985380864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320266.972780:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380736 00000100:00100000:1.0:1767320266.972786:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380736 00000100:00100000:0.0:1767320266.972803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320266.972812:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5762 00000100:00100000:1.0:1767320266.972818:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985380736:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320266.972829:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985381120:0@lo:38:kworker.0 00010000:00020000:1.0:1767320266.972839:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320266.972851:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320266.972875:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985381376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320266.972882:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985380736:12345-0@lo:38:kworker.0 Request processed in 60us (137us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320266.972890:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5762 00000100:00100000:1.0:1767320266.972902:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985381120 00000100:00100000:0.0:1767320266.972903:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320266.972908:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985381120 00000100:00080000:0.0:1767320266.972922:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320266.972925:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985381376 00000100:00100000:1.0:1767320266.972928:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985381376 00000100:00080000:0.0:1767320266.972941:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320266.972943:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5763 00000100:00100000:0.0:1767320266.972948:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985374208:0@lo:250:kworker.0 00000100:00100000:1.0:1767320266.972951:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985381120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320266.972964:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320266.972988:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320266.972994:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320266.972994:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985381120:12345-0@lo:38:kworker.0 Request processed in 43us (142us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320266.973002:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985380736:0@lo:38:kworker.0 00000100:00080000:0.0:1767320266.973013:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320266.973016:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320266.973019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985381120:0@lo:38:kworker.0 00000100:00080000:0.0:1767320266.973038:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320266.973042:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320266.973044:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320266.973048:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5489 00000100:00080000:0.0:1767320266.973051:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320266.973055:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320266.973072:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2d80 x1853163985381760/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320266.973107:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985381760:0@lo:250:kworker.0 00000100:00100000:2.0:1767320266.973139:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985381504:0@lo:400:kworker.0 00000100:00100000:3.0:1767320266.973190:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985380864 00000100:00100000:2.0:1767320266.973193:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5764 00000100:00100000:3.0:1767320266.973196:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985380864 00000100:00100000:2.0:1767320266.973202:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985381376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320266.973213:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320266.973224:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1931 00000100:00100000:3.0:1767320266.973230:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985380864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320266.973253:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320266.973258:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5763 00000100:00100000:2.0:1767320266.977287:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985381376:12345-0@lo:38:kworker.0 Request processed in 4075us (4374us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320266.977303:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5764 00000100:00100000:3.0:1767320266.977544:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985380864:12345-0@lo:38:kworker.0 Request processed in 4311us (4739us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320266.977554:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1931 00000400:00000100:0.0:1767320266.977618:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985381760 offset 0 length 520: 4 00000100:00080000:0.0:1767320266.977913:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320266.977919:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320266.977923:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985380864:0@lo:38:kworker.0 00000100:00080000:0.0:1767320266.977937:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320266.978116:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320266.978120:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985381376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320267.486616:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985382016:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320267.486695:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320267.486752:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320267.486760:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985381888:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320267.486781:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320267.486886:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382016 00000100:00100000:3.0:1767320267.486895:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382016 00000100:00100000:3.0:1767320267.486945:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985381888 00000100:00100000:3.0:1767320267.486948:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985381888 00000100:00100000:3.0:1767320267.486967:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2528 00000100:00100000:3.0:1767320267.486974:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985382016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320267.487020:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320267.487068:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985382016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (373us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320267.487075:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2528 00000100:00100000:3.0:1767320267.487092:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2529 00000100:00100000:3.0:1767320267.487098:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985381888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320267.487112:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320267.487136:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985381888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 39us (354us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320267.487142:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2529 00000100:00100000:3.0:1767320267.487214:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985381888:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320267.487239:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985382016:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320267.612301:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320268.636748:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320269.659406:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320270.683606:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320271.018879:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320271.018934:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320271.026800:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920607744 00000100:00100000:1.0:1767320271.026816:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920607744 00000100:00100000:1.0:1767320271.026889:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920607616 00000100:00100000:1.0:1767320271.026892:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920607616 00000100:00100000:1.0:1767320271.026914:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5120 00000100:00100000:1.0:1767320271.026924:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920607744:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320271.027007:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920607744:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (8138us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320271.027015:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5120 00000100:00100000:1.0:1767320271.027034:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5765 00000100:00100000:1.0:1767320271.027039:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920607616:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320271.027058:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320271.027429:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320271.027696:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920607616:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 654us (8760us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320271.027705:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5765 00010000:00010000:1.0:1767320271.707422:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320272.091775:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320272.091785:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320272.091788:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320272.091792:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320272.091796:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320272.091803:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5515 00000100:00080000:1.0:1767320272.091807:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320272.091813:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320272.091850:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552f100 x1853163985382144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320272.091873:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320272.091876:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320272.091878:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320272.091881:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5515 00000100:00080000:1.0:1767320272.091883:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320272.091887:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320272.091897:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552d880 x1853163985382272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320272.091935:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320272.091939:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320272.091941:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320272.091944:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5515 00000100:00080000:1.0:1767320272.091946:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320272.091949:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320272.091960:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985382528/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320272.091975:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320272.091978:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320272.091979:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320272.091982:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5515 00000100:00080000:1.0:1767320272.091985:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320272.091989:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320272.091990:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985382400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320272.091999:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2d80 x1853163985382784/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320272.092048:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985382912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320272.092058:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320272.092087:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320272.092126:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320272.092132:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985382656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320272.092154:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985382144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320272.092155:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320272.092229:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382400 00000100:00100000:0.0:1767320272.092237:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382400 00000100:00100000:2.0:1767320272.092242:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382912 00000100:00100000:2.0:1767320272.092247:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382912 00000100:00100000:0.0:1767320272.092285:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382656 00000100:00100000:0.0:1767320272.092289:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382656 00000100:00100000:2.0:1767320272.092306:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3333 00000100:00100000:2.0:1767320272.092318:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985382912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320272.092343:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5121 00000100:00100000:0.0:1767320272.092351:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985382400:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320272.092376:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985382912:12345-0@lo:400:kworker.0 Request processed in 59us (288us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320272.092384:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3333 00000100:00100000:0.0:1767320272.092405:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985382400:12345-0@lo:400:kworker.0 Request processed in 54us (346us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320272.092413:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5121 00000100:00100000:0.0:1767320272.092430:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5122 00000100:00100000:0.0:1767320272.092437:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985382656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320272.092441:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985382400:0@lo:400:kworker.0 00000100:00100000:0.0:1767320272.092480:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985382656:12345-0@lo:400:kworker.0 Request processed in 42us (324us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320272.092486:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5122 00000100:00100000:2.0:1767320272.092489:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985382656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320272.092509:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320272.092527:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382144 00000100:00100000:2.0:1767320272.092532:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382144 00000100:00100000:0.0:1767320272.092534:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985382272:0@lo:38:kworker.0 00000100:00100000:0.0:1767320272.092561:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320272.092566:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5766 00000100:00100000:2.0:1767320272.092576:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985382144:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320272.092582:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985382528:0@lo:38:kworker.0 00010000:00020000:2.0:1767320272.092594:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320272.092600:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320272.092616:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985382784:0@lo:38:kworker.0 00000100:00100000:2.0:1767320272.092632:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985382144:12345-0@lo:38:kworker.0 Request processed in 55us (122us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320272.092641:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5766 00000100:00100000:0.0:1767320272.092642:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320272.092658:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382528 00000100:00100000:2.0:1767320272.092662:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382528 00000100:00080000:0.0:1767320272.092673:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767320272.092675:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382784 00000100:00100000:2.0:1767320272.092679:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382784 00000100:00080000:0.0:1767320272.092681:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320272.092687:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985382144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320272.092694:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5767 00000100:00100000:2.0:1767320272.092699:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985382528:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320272.092706:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320272.092724:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5768 00000100:00100000:2.0:1767320272.092731:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985382528:12345-0@lo:38:kworker.0 Request processed in 31us (130us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320272.092733:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985382784:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320272.092745:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5767 00010000:00020000:0.0:1767320272.092752:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320272.092778:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985382784:12345-0@lo:38:kworker.0 Request processed in 47us (140us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320272.092783:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5768 00000100:00080000:0.0:1767320272.092805:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320272.092808:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320272.092811:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985382528:0@lo:38:kworker.0 00000100:00080000:0.0:1767320272.092816:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320272.092818:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320272.092821:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985382784:0@lo:38:kworker.0 00000100:00100000:3.0:1767320272.093031:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985383040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320272.093061:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320272.093100:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985382272 00000100:00100000:3.0:1767320272.093104:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985382272 00000100:00100000:3.0:1767320272.093135:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1932 00000100:00100000:3.0:1767320272.093140:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985382272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320272.093150:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320272.093187:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383040 00000100:00100000:2.0:1767320272.093193:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383040 00000100:00100000:3.0:1767320272.093200:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985382272:12345-0@lo:38:kworker.0 Request processed in 59us (639us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320272.093209:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1932 00000100:00080000:3.0:1767320272.093242:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320272.093246:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3334 00000100:00080000:3.0:1767320272.093249:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320272.093252:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320272.093258:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985383040:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320272.093363:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985383040:12345-0@lo:400:kworker.0 Request processed in 108us (300us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320272.093370:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3334 00000100:00080000:0.0:1767320272.093404:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320272.093409:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320272.093412:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985382272:0@lo:38:kworker.0 00000100:00100000:3.0:1767320272.096275:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985383040:0@lo:400:kworker.0 00000100:00100000:1.0:1767320272.096963:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985382912:0@lo:400:kworker.0 00000100:00100000:3.0:1767320272.604689:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320272.604711:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985383168:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320272.604773:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320272.604840:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383168 00000100:00100000:1.0:1767320272.604849:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383168 00000100:00100000:1.0:1767320272.604953:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2530 00000100:00100000:1.0:1767320272.604963:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985383168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320272.604989:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320272.605034:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985383168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320272.605042:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2530 00000100:00100000:2.0:1767320272.605302:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985383168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320272.607540:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320272.607563:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985383296:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320272.607627:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320272.608471:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383296 00000100:00100000:3.0:1767320272.608482:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383296 00000100:00100000:3.0:1767320272.608538:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2531 00000100:00100000:3.0:1767320272.608547:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985383296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320272.608574:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320272.608626:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985383296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (998us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320272.608633:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2531 00000100:00100000:3.0:1767320272.608689:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985383296:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320272.731507:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320273.756322:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320274.780327:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320275.803437:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320276.137611:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320276.137744:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920608000 00000100:00100000:3.0:1767320276.137755:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920608000 00000100:00100000:3.0:1767320276.137842:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5123 00000100:00100000:3.0:1767320276.137852:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920608000:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320276.137929:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920608000:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (331us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320276.137936:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5123 00000100:00080000:0.0:1767320276.138057:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320276.145887:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320276.146010:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920607872 00000100:00100000:0.0:1767320276.146021:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920607872 00000100:00100000:0.0:1767320276.146069:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5769 00000100:00100000:0.0:1767320276.146077:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920607872:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320276.146100:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320276.146205:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920607872:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 84us (283us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320276.146213:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5769 00010000:00010000:0.0:1767320276.829384:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320277.211330:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320277.211342:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320277.211345:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320277.211349:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320277.211353:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320277.211361:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5520 00000100:00080000:0.0:1767320277.211366:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320277.211374:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320277.211429:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0000 x1853163985383424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320277.211446:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320277.211450:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320277.211452:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320277.211455:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5520 00000100:00080000:0.0:1767320277.211457:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320277.211461:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320277.211475:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0a80 x1853163985383552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320277.211520:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320277.211524:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320277.211526:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320277.211530:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5520 00000100:00080000:0.0:1767320277.211533:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320277.211536:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320277.211550:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0700 x1853163985383808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320277.211568:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320277.211571:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320277.211573:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320277.211576:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5520 00000100:00080000:0.0:1767320277.211579:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320277.211582:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320277.211593:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb100 x1853163985384064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320277.211654:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985384320:0@lo:400:kworker.0 00000100:00100000:1.0:1767320277.211677:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985383680:0@lo:400:kworker.0 00000100:00100000:0.0:1767320277.211733:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320277.211733:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320277.211837:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384320 00000100:00100000:2.0:1767320277.211845:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384320 00000100:00100000:2.0:1767320277.211900:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3335 00000100:00100000:2.0:1767320277.211910:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985384320:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320277.211960:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985384320:12345-0@lo:400:kworker.0 Request processed in 52us (234us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320277.211967:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3335 00000100:00100000:0.0:1767320277.212046:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985384320:0@lo:400:kworker.0 00000100:00100000:0.0:1767320277.212078:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320277.212085:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985384192:0@lo:400:kworker.0 00000100:00100000:0.0:1767320277.212110:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320277.212192:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384192 00000100:00100000:3.0:1767320277.212198:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384192 00000100:00100000:3.0:1767320277.212238:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3336 00000100:00100000:3.0:1767320277.212245:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985384192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320277.212283:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985384192:12345-0@lo:400:kworker.0 Request processed in 39us (173us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320277.212290:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3336 00000100:00100000:3.0:1767320277.212308:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985383936:0@lo:400:kworker.0 00000100:00100000:3.0:1767320277.212332:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320277.212403:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985383424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320277.212433:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320277.212500:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985384192:0@lo:400:kworker.0 00000100:00100000:0.0:1767320277.212531:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383680 00000100:00100000:0.0:1767320277.212536:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383680 00000100:00100000:0.0:1767320277.212563:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383936 00000100:00100000:0.0:1767320277.212566:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383936 00000100:00100000:0.0:1767320277.212585:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383424 00000100:00100000:0.0:1767320277.212588:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383424 00000100:00100000:0.0:1767320277.212601:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5124 00000100:00100000:0.0:1767320277.212607:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985383680:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320277.212639:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985383680:12345-0@lo:400:kworker.0 Request processed in 32us (914us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320277.212644:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5124 00000100:00100000:0.0:1767320277.212658:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5125 00000100:00100000:0.0:1767320277.212664:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985383936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320277.212687:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985383936:12345-0@lo:400:kworker.0 Request processed in 24us (355us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320277.212696:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5125 00000100:00100000:0.0:1767320277.212706:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5770 00000100:00100000:0.0:1767320277.212709:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985383424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320277.212725:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320277.212752:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985383424:12345-0@lo:38:kworker.0 Request processed in 41us (318us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320277.212758:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5770 00000100:00080000:0.0:1767320277.212802:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320277.212806:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320277.212808:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320277.212886:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985383936:0@lo:400:kworker.0 00000100:00100000:1.0:1767320277.213353:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985383680:0@lo:400:kworker.0 00000100:00080000:2.0:1767320277.213356:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320277.213364:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320277.213368:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985383424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320277.213377:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985383552:0@lo:38:kworker.0 00000100:00100000:2.0:1767320277.213407:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320277.213429:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985383808:0@lo:38:kworker.0 00000100:00100000:2.0:1767320277.213450:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320277.213537:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383552 00000100:00100000:0.0:1767320277.213542:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383552 00000100:00100000:0.0:1767320277.213577:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1933 00000100:00100000:0.0:1767320277.213583:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985383552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320277.213595:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320277.213625:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985383552:12345-0@lo:38:kworker.0 Request processed in 41us (216us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320277.213631:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1933 00000100:00100000:2.0:1767320277.213689:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985384064:0@lo:38:kworker.0 00000100:00100000:2.0:1767320277.213713:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320277.213933:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320277.213938:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320277.213941:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985383552:0@lo:38:kworker.0 00000100:00100000:3.0:1767320277.213983:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985383808 00000100:00100000:3.0:1767320277.213988:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985383808 00000100:00100000:0.0:1767320277.214021:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384064 00000100:00100000:0.0:1767320277.214024:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384064 00000100:00100000:0.0:1767320277.214061:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5771 00000100:00100000:0.0:1767320277.214066:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985383808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320277.214076:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320277.214114:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985383808:12345-0@lo:38:kworker.0 Request processed in 47us (662us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320277.214120:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5771 00000100:00100000:0.0:1767320277.214134:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5772 00000100:00100000:0.0:1767320277.214137:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985384064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320277.214145:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320277.214188:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985384064:12345-0@lo:38:kworker.0 Request processed in 49us (474us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320277.214193:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5772 00000100:00080000:2.0:1767320277.214304:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320277.214310:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320277.214312:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985383808:0@lo:38:kworker.0 00000100:00080000:2.0:1767320277.214320:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320277.214323:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320277.214325:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985384064:0@lo:38:kworker.0 00000100:00100000:1.0:1767320277.726086:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985384448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320277.726161:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320277.726403:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384448 00000100:00100000:3.0:1767320277.726413:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384448 00000100:00100000:3.0:1767320277.726474:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2532 00000100:00100000:3.0:1767320277.726483:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985384448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320277.726519:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320277.726562:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985384448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320277.726570:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2532 00000100:00100000:2.0:1767320277.727213:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985384576:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320277.727248:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320277.727407:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985384448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320277.729614:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384576 00000100:00100000:0.0:1767320277.729630:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384576 00000100:00100000:0.0:1767320277.729697:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2533 00000100:00100000:0.0:1767320277.729707:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985384576:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320277.729739:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320277.730001:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985384576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 294us (2751us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320277.730019:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2533 00000100:00100000:2.0:1767320277.730315:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985384576:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320277.852945:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320278.875748:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320279.899851:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320280.924498:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320281.258317:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320281.258546:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920608256 00000100:00100000:3.0:1767320281.258558:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920608256 00000100:00100000:3.0:1767320281.258644:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5126 00000100:00100000:3.0:1767320281.258655:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920608256:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767320281.258912:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320281.259137:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920608256:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 474us (822us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320281.259146:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5126 00000100:00100000:3.0:1767320281.261410:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320281.266092:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920608128 00000100:00100000:0.0:1767320281.266108:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920608128 00000100:00100000:0.0:1767320281.266277:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5773 00000100:00100000:0.0:1767320281.266287:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920608128:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320281.266311:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320281.266390:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920608128:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 103us (4990us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320281.266398:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5773 00010000:00010000:0.0:1767320281.950038:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320282.331353:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320282.331364:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320282.331367:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320282.331370:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320282.331375:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320282.331383:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5525 00000100:00080000:1.0:1767320282.331387:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320282.331394:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320282.331451:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3b80 x1853163985384704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320282.331476:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320282.331479:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320282.331481:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320282.331484:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5525 00000100:00080000:1.0:1767320282.331487:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320282.331491:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320282.331503:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2d80 x1853163985384832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320282.331554:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320282.331558:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320282.331560:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320282.331564:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5525 00000100:00080000:1.0:1767320282.331567:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320282.331571:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320282.331583:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1180 x1853163985385088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320282.331601:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985384704:0@lo:38:kworker.0 00000100:00080000:1.0:1767320282.331603:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320282.331611:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320282.331613:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320282.331616:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5525 00000100:00080000:1.0:1767320282.331619:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320282.331622:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320282.331633:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0a80 x1853163985385344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320282.331678:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320282.331688:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985385472:0@lo:400:kworker.0 00000100:00100000:1.0:1767320282.331729:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320282.331746:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985384832:0@lo:38:kworker.0 00000100:00100000:1.0:1767320282.331762:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320282.331770:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985385600:0@lo:400:kworker.0 00000100:00100000:0.0:1767320282.331777:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320282.331801:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320282.331817:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985385088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320282.331843:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320282.331854:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385472 00000100:00100000:0.0:1767320282.331867:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985385344:0@lo:38:kworker.0 00000100:00100000:1.0:1767320282.331872:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385472 00000100:00100000:3.0:1767320282.331881:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985384960:0@lo:400:kworker.0 00000100:00100000:0.0:1767320282.331893:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320282.331925:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320282.331938:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384832 00000100:00100000:0.0:1767320282.331943:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384832 00000100:00100000:3.0:1767320282.331954:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320282.331962:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985385216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320282.331975:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1934 00000100:00100000:0.0:1767320282.331983:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985384832:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320282.331989:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767320282.332008:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320282.332019:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384704 00000100:00100000:3.0:1767320282.332024:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384704 00000100:00100000:3.0:1767320282.332047:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385088 00000100:00100000:3.0:1767320282.332049:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385088 00000100:00100000:3.0:1767320282.332059:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385344 00000100:00100000:3.0:1767320282.332062:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385344 00000100:00100000:0.0:1767320282.332067:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985384832:12345-0@lo:38:kworker.0 Request processed in 83us (289us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320282.332071:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985384960 00000100:00100000:3.0:1767320282.332074:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985384960 00000100:00100000:0.0:1767320282.332078:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1934 00000100:00100000:3.0:1767320282.332097:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385216 00000100:00100000:3.0:1767320282.332101:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385216 00000100:00100000:3.0:1767320282.332128:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5127 00000100:00100000:0.0:1767320282.332135:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385600 00000100:00100000:3.0:1767320282.332136:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985384960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320282.332140:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385600 00000100:00100000:1.0:1767320282.332158:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3337 00000100:00100000:1.0:1767320282.332199:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985385472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320282.332202:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3338 00000100:00100000:0.0:1767320282.332214:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985385600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320282.332221:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985384960:12345-0@lo:400:kworker.0 Request processed in 83us (294us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320282.332229:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5127 00000100:00100000:3.0:1767320282.332249:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5128 00000100:00100000:1.0:1767320282.332256:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985385472:12345-0@lo:400:kworker.0 Request processed in 60us (527us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320282.332261:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985385216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320282.332265:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985385600:12345-0@lo:400:kworker.0 Request processed in 51us (463us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320282.332273:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3337 00000100:00100000:0.0:1767320282.332280:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3338 00000100:00100000:3.0:1767320282.332303:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985385216:12345-0@lo:400:kworker.0 Request processed in 45us (315us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320282.332305:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5774 00000100:00100000:0.0:1767320282.332317:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5775 00000100:00100000:1.0:1767320282.332318:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985384704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320282.332330:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320282.332359:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985384704:12345-0@lo:38:kworker.0 Request processed in 45us (682us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320282.332480:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985385088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320282.332492:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320282.332518:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985385088:12345-0@lo:38:kworker.0 Request processed in 38us (674us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320282.332871:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5128 00000100:00100000:3.0:1767320282.332891:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5776 00000100:00100000:3.0:1767320282.332895:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985385344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320282.332906:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320282.332933:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985385344:12345-0@lo:38:kworker.0 Request processed in 36us (1041us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320282.332939:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5776 00000100:00100000:3.0:1767320282.332983:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985385216:0@lo:400:kworker.0 00000100:00100000:3.0:1767320282.332994:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985384960:0@lo:400:kworker.0 00000100:00080000:3.0:1767320282.333047:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320282.333052:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320282.333053:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320282.333075:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320282.333080:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320282.333083:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985385088:0@lo:38:kworker.0 00000100:00080000:3.0:1767320282.333091:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320282.333094:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320282.333097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985385344:0@lo:38:kworker.0 00000100:00080000:3.0:1767320282.333119:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320282.333124:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320282.333126:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985384704:0@lo:38:kworker.0 00000100:00080000:3.0:1767320282.333136:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320282.333139:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320282.333142:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985384832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320282.333205:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985385600:0@lo:400:kworker.0 00000100:00100000:3.0:1767320282.333216:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985385472:0@lo:400:kworker.0 00000100:00100000:1.0:1767320282.334455:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5774 00000100:00100000:0.0:1767320282.342221:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5775 00000100:00100000:3.0:1767320282.843458:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985385728:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320282.843514:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320282.844677:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385728 00000100:00100000:1.0:1767320282.844690:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385728 00000100:00100000:1.0:1767320282.844752:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2534 00000100:00100000:1.0:1767320282.844762:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985385728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320282.844793:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320282.844809:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985385856:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320282.844899:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985385728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320282.845464:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320282.845573:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385856 00000100:00100000:3.0:1767320282.845579:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385856 00000100:00100000:3.0:1767320282.845656:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2535 00000100:00100000:3.0:1767320282.845664:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985385856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320282.845682:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320282.845763:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985385856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (299us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320282.845771:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2535 00000100:00100000:2.0:1767320282.845951:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985385856:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320282.851967:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985385728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 7194us (8438us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320282.851983:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2534 00010000:00010000:2.0:1767320282.971642:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320284.000897:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320285.020213:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320286.046234:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320286.383682:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320286.383738:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320286.383770:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920608384 00000100:00100000:2.0:1767320286.383781:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920608384 00000100:00100000:2.0:1767320286.383847:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920608512 00000100:00100000:2.0:1767320286.383851:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920608512 00000100:00100000:2.0:1767320286.383896:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5129 00000100:00100000:2.0:1767320286.383905:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920608512:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767320286.383944:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320286.383973:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920608512:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 69us (234us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320286.383982:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5129 00000100:00100000:2.0:1767320286.384002:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5777 00000100:00100000:2.0:1767320286.384007:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920608384:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320286.384028:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320286.384077:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920608384:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 67us (405us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320286.384085:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5777 00010000:00010000:1.0:1767320287.076643:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320287.451326:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320287.451342:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320287.451346:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320287.451350:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320287.451354:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320287.451362:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5530 00000100:00080000:1.0:1767320287.451366:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320287.451373:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320287.451414:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10e00 x1853163985385984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320287.451440:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320287.451444:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320287.451445:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320287.451448:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5530 00000100:00080000:1.0:1767320287.451451:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320287.451453:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320287.451465:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11880 x1853163985386112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320287.451499:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320287.451502:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320287.451504:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320287.451508:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5530 00000100:00080000:1.0:1767320287.451510:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320287.451513:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320287.451521:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985386368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320287.451535:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320287.451538:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320287.451540:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320287.451543:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5530 00000100:00080000:1.0:1767320287.451545:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320287.451548:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320287.451557:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12d80 x1853163985386624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320287.451617:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985386880:0@lo:400:kworker.0 00000100:00100000:0.0:1767320287.451620:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985385984:0@lo:38:kworker.0 00000100:00100000:2.0:1767320287.451718:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985386496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320287.451732:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985386752:0@lo:400:kworker.0 00000100:00100000:0.0:1767320287.453955:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320287.454043:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985386112:0@lo:38:kworker.0 00000100:00100000:1.0:1767320287.454072:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320287.454087:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320287.456659:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985386112 00000100:00100000:1.0:1767320287.456674:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985386112 00000100:00100000:1.0:1767320287.456719:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1935 00000100:00100000:1.0:1767320287.456727:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985386112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320287.456750:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320287.456810:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985386112:12345-0@lo:38:kworker.0 Request processed in 83us (2723us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320287.456817:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1935 00000100:00100000:1.0:1767320287.456870:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985386240:0@lo:400:kworker.0 00000100:00100000:1.0:1767320287.457110:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320287.457154:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985385984 00000100:00100000:1.0:1767320287.457159:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985385984 00000100:00100000:1.0:1767320287.457197:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985386240 00000100:00100000:1.0:1767320287.457201:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985386240 00000100:00100000:1.0:1767320287.457247:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5130 00000100:00100000:1.0:1767320287.457253:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985386240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320287.457294:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985386240:12345-0@lo:400:kworker.0 Request processed in 42us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320287.457506:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5130 00000100:00100000:1.0:1767320287.457521:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5778 00000100:00100000:1.0:1767320287.457525:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985385984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320287.457539:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320287.457567:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985385984:12345-0@lo:38:kworker.0 Request processed in 40us (3618us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320287.457572:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5778 00000100:00100000:1.0:1767320287.457613:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985386240:0@lo:400:kworker.0 00000100:00080000:1.0:1767320287.457647:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320287.458045:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320287.458097:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985386880 00000100:00100000:2.0:1767320287.458102:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985386880 00000100:00100000:2.0:1767320287.458140:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3339 00000100:00100000:2.0:1767320287.458148:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985386880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320287.458214:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985386880:12345-0@lo:400:kworker.0 Request processed in 68us (4143us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320287.458221:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3339 00000100:00080000:2.0:1767320287.458240:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320287.458261:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985386880:0@lo:400:kworker.0 00000100:00080000:0.0:1767320287.459861:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320287.459872:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320287.459877:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985386112:0@lo:38:kworker.0 00000100:00100000:0.0:1767320287.459888:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985386368:0@lo:38:kworker.0 00000100:00100000:1.0:1767320287.459896:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985386496 00000100:00100000:1.0:1767320287.459952:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985386496 00000100:00080000:2.0:1767320287.459985:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320287.460027:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5131 00000100:00100000:1.0:1767320287.460036:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985386496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320287.460112:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985386496:12345-0@lo:400:kworker.0 Request processed in 77us (2076us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320287.460118:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5131 00000100:00100000:0.0:1767320287.460157:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320287.460218:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320287.460284:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985386496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320287.460546:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985386752 00000100:00100000:1.0:1767320287.460551:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985386752 00000100:00100000:1.0:1767320287.460624:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3340 00000100:00100000:1.0:1767320287.460631:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985386752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320287.460669:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985386752:12345-0@lo:400:kworker.0 Request processed in 39us (450us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320287.460675:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3340 00000100:00080000:3.0:1767320287.460714:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320287.460735:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985386752:0@lo:400:kworker.0 00000100:00100000:1.0:1767320287.461598:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985386368 00000100:00100000:1.0:1767320287.461606:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985386368 00000100:00100000:0.0:1767320287.461608:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985386624:0@lo:38:kworker.0 00000100:00100000:0.0:1767320287.461643:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320287.461648:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5779 00000100:00100000:1.0:1767320287.461655:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985386368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320287.461677:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320287.461714:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985386368:12345-0@lo:38:kworker.0 Request processed in 59us (1558us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320287.461721:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5779 00000100:00080000:0.0:1767320287.461998:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320287.462003:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320287.462007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985385984:0@lo:38:kworker.0 00000100:00080000:0.0:1767320287.462026:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320287.462030:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320287.462032:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985386368:0@lo:38:kworker.0 00000100:00100000:1.0:1767320287.463794:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985386624 00000100:00100000:1.0:1767320287.463806:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985386624 00000100:00100000:1.0:1767320287.463841:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5780 00000100:00100000:1.0:1767320287.463848:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985386624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320287.463867:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320287.463940:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985386624:12345-0@lo:38:kworker.0 Request processed in 92us (2297us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320287.463947:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5780 00000100:00080000:0.0:1767320287.464076:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320287.464081:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320287.464085:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985386624:0@lo:38:kworker.0 00000100:00100000:1.0:1767320287.963448:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985387008:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320287.963630:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320287.963723:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985387136:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320287.963755:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320287.964704:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985387008 00000100:00100000:2.0:1767320287.964716:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985387008 00000100:00100000:2.0:1767320287.964817:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985387136 00000100:00100000:2.0:1767320287.964830:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985387136 00000100:00100000:0.0:1767320287.964843:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2536 00000100:00100000:0.0:1767320287.964857:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985387008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320287.964887:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320287.964952:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2537 00000100:00100000:3.0:1767320287.964971:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985387136:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320287.964994:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985387008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 140us (1474us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320287.965010:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2536 00002000:00000024:3.0:1767320287.965047:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320287.965288:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985387008:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320287.965318:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985387136:12345-0@lo:13:osp-pre-0-1.0 Request processed in 348us (1561us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320287.965328:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2537 00000100:00100000:0.0:1767320287.965526:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985387136:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320288.091558:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320289.116691:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320290.139520:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320290.474198:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767320290.475159:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920608640 offset 0 length 520: 4 00000100:00100000:1.0:1767320290.475317:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920608896 00000100:00100000:1.0:1767320290.475328:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920608896 00000100:00100000:1.0:1767320290.475457:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5132 00000100:00100000:1.0:1767320290.475467:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920608896:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320290.475544:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920608896:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (1388us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320290.475553:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5132 00000100:00080000:3.0:1767320290.477387:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320290.480803:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320290.488171:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920608768 00000100:00100000:0.0:1767320290.488508:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920608768 00000100:00100000:3.0:1767320290.488727:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5781 00000100:00100000:3.0:1767320290.488737:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920608768:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320290.488762:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320290.488938:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920608768:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 199us (8145us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320290.488946:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5781 00010000:00010000:1.0:1767320291.163763:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320291.547332:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320266/real 1767320266] req@ffff8ce1b3ff2d80 x1853163985381760/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320291 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320291.547364:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3ff2d80 x1853163985381760/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320291 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320291.547379:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320291.547400:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320291.547402:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985381760:0@lo:250:kworker.0 00000100:00080000:2.0:1767320291.547549:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320291.547556:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320291.547559:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320291.547563:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5515 00000100:00080000:2.0:1767320291.547568:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320291.547573:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320291.547602:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163985387264/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320291.547621:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320291.547624:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320291.547625:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320291.547627:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5535 00000100:00080000:2.0:1767320291.547628:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320291.547630:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320291.547639:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ce00 x1853163985387392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320291.547644:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320291.547646:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320291.547648:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320291.547649:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5535 00000100:00080000:2.0:1767320291.547651:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320291.547653:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320291.547659:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c380 x1853163985387520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320291.547683:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320291.547685:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320291.547686:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320291.547688:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5535 00000100:00080000:2.0:1767320291.547690:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320291.547691:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320291.547697:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f480 x1853163985387776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320291.547737:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320291.547739:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320291.547741:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320291.547745:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5535 00000100:00080000:2.0:1767320291.547746:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320291.547749:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320291.547759:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af480 x1853163985388032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320291.547799:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985387904:0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.547855:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320291.547906:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320291.547912:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985387648:0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.547935:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320291.548020:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985388288:0@lo:400:kworker.0 00000100:00100000:0.0:1767320291.548049:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320291.548077:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320291.548082:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985388160:0@lo:400:kworker.0 00000100:00100000:0.0:1767320291.548101:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320291.548122:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985387264:0@lo:250:kworker.0 00000400:00000100:0.0:1767320291.548139:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985387264 offset 0 length 520: 4 00000100:00100000:0.0:1767320291.548156:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985387392:0@lo:38:kworker.0 00000100:00100000:0.0:1767320291.548201:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320291.548216:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985387520:0@lo:38:kworker.0 00000100:00100000:0.0:1767320291.548235:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320291.548249:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985387776:0@lo:38:kworker.0 00000100:00100000:0.0:1767320291.548265:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320291.548279:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985388032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320291.548387:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388288 00000100:00100000:2.0:1767320291.548394:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388288 00000100:00100000:2.0:1767320291.548442:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388160 00000100:00100000:2.0:1767320291.548446:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388160 00000100:00100000:2.0:1767320291.548466:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3341 00000100:00100000:2.0:1767320291.548474:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985388288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.548522:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985388288:12345-0@lo:400:kworker.0 Request processed in 48us (471us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320291.548529:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3341 00000100:00100000:2.0:1767320291.548545:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3342 00000100:00100000:2.0:1767320291.548551:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985388160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.548575:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985388160:12345-0@lo:400:kworker.0 Request processed in 25us (473us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320291.548581:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3342 00000100:00100000:2.0:1767320291.548615:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985387904 00000100:00100000:2.0:1767320291.548619:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985387904 00000100:00100000:2.0:1767320291.548641:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985387648 00000100:00100000:2.0:1767320291.548643:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985387648 00000100:00100000:2.0:1767320291.548658:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985387392 00000100:00100000:2.0:1767320291.548661:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985387392 00000100:00100000:2.0:1767320291.548669:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985387776 00000100:00100000:2.0:1767320291.548671:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985387776 00000100:00100000:2.0:1767320291.548679:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5133 00000100:00100000:2.0:1767320291.548685:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985387904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.548712:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985387904:12345-0@lo:400:kworker.0 Request processed in 28us (858us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320291.548715:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5133 00000100:00100000:2.0:1767320291.548737:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985387904:0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.548750:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5134 00000100:00100000:2.0:1767320291.548754:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985387648:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.548772:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985387648:12345-0@lo:400:kworker.0 Request processed in 18us (837us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320291.548775:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5134 00000100:00100000:2.0:1767320291.548787:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985387648:0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.548797:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5782 00000100:00100000:2.0:1767320291.548800:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985387392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320291.548812:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320291.548828:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985387392:12345-0@lo:38:kworker.0 Request processed in 27us (628us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320291.548832:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5782 00000100:00100000:2.0:1767320291.548838:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5783 00000100:00100000:2.0:1767320291.548840:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985387776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320291.548845:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320291.548856:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985387776:12345-0@lo:38:kworker.0 Request processed in 15us (590us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320291.548859:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5783 00000100:00100000:2.0:1767320291.548878:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985388160:0@lo:400:kworker.0 00000100:00100000:2.0:1767320291.548887:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985388288:0@lo:400:kworker.0 00000100:00080000:3.0:1767320291.548972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320291.548977:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320291.548979:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320291.549041:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320291.549076:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320291.549079:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320291.549082:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985387392:0@lo:38:kworker.0 00000100:00080000:0.0:1767320291.549088:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320291.549090:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320291.549091:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985387776:0@lo:38:kworker.0 00000100:00100000:1.0:1767320291.549160:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388032 00000100:00100000:1.0:1767320291.549188:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388032 00000100:00100000:1.0:1767320291.549216:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5784 00000100:00100000:1.0:1767320291.549222:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985388032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320291.549232:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320291.549263:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985388032:12345-0@lo:38:kworker.0 Request processed in 40us (222us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320291.549268:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5784 00000100:00100000:1.0:1767320291.549363:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985387520 00000100:00100000:1.0:1767320291.549366:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985387520 00000100:00100000:1.0:1767320291.549382:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1936 00000100:00100000:1.0:1767320291.549385:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985387520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320291.549392:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320291.549410:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985387520:12345-0@lo:38:kworker.0 Request processed in 24us (1175us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320291.549414:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1936 00000100:00080000:0.0:1767320291.549517:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320291.549521:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320291.549524:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985387520:0@lo:38:kworker.0 00000100:00080000:0.0:1767320291.549533:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320291.549536:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320291.549538:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985388032:0@lo:38:kworker.0 00010000:00010000:2.0:1767320292.188014:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320293.083577:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320293.083601:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985388416:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320293.083695:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320293.083875:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388416 00000100:00100000:0.0:1767320293.083884:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388416 00000100:00100000:0.0:1767320293.083969:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2538 00000100:00100000:0.0:1767320293.083980:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985388416:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320293.084014:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320293.084060:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985388416:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (366us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320293.084068:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2538 00000100:00080000:1.0:1767320293.084150:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320293.084210:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985388416:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320293.084497:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985388544:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320293.084526:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320293.084645:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388544 00000100:00100000:2.0:1767320293.084651:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388544 00000100:00100000:2.0:1767320293.084693:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2539 00000100:00100000:2.0:1767320293.084701:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985388544:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320293.084812:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767320293.084868:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320293.084922:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985388544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 221us (395us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320293.084928:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2539 00000100:00100000:1.0:1767320293.084990:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985388544:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320293.211958:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320294.235765:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320295.265201:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320295.600489:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320295.600541:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320295.606508:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609152 00000100:00100000:3.0:1767320295.606522:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609152 00000100:00100000:3.0:1767320295.606679:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609024 00000100:00100000:3.0:1767320295.606683:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609024 00000100:00100000:3.0:1767320295.606704:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5135 00000100:00100000:3.0:1767320295.606714:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920609152:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320295.606796:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920609152:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (6317us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320295.606804:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5135 00000100:00100000:0.0:1767320295.606819:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5785 00000100:00100000:0.0:1767320295.606829:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920609024:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320295.606850:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320295.606892:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920609024:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 63us (6351us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320295.606898:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5785 00000100:00080000:2.0:1767320295.607696:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320296.286803:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320296.667299:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320296.667311:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320296.667314:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320296.667319:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320296.667324:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320296.667333:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5539 00000100:00080000:0.0:1767320296.667338:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320296.667344:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320296.667389:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c000 x1853163985388672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320296.667415:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320296.667419:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320296.667420:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320296.667423:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5539 00000100:00080000:0.0:1767320296.667426:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320296.667430:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320296.667442:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ca80 x1853163985388800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320296.667529:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985388928:0@lo:400:kworker.0 00000100:00100000:2.0:1767320296.667594:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985388672:0@lo:38:kworker.0 00000100:00100000:1.0:1767320296.667606:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320296.667645:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320296.667697:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985388800:0@lo:38:kworker.0 00000100:00080000:0.0:1767320296.667724:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320296.667730:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320296.667734:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320296.667734:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320296.667741:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388928 00000100:00080000:0.0:1767320296.667743:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5539 00000100:00080000:0.0:1767320296.667751:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320296.667758:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320296.667760:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388928 00000100:00100000:0.0:1767320296.667777:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985389056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320296.667803:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388800 00000100:00100000:2.0:1767320296.667813:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388800 00000100:00100000:2.0:1767320296.667868:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1937 00000100:00100000:2.0:1767320296.667876:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985388800:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320296.667894:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985388672 00010000:00020000:2.0:1767320296.667902:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320296.667902:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985388672 00000100:00100000:3.0:1767320296.667935:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5136 00000100:00100000:2.0:1767320296.667948:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985388800:12345-0@lo:38:kworker.0 Request processed in 67us (213us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320296.667956:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985388928:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320296.667968:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1937 00000100:00100000:2.0:1767320296.668004:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985389184:0@lo:400:kworker.0 00000100:00100000:2.0:1767320296.668057:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320296.668086:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320296.668098:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985389056:0@lo:38:kworker.0 00000100:00100000:2.0:1767320296.668117:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320296.668145:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320296.668150:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320296.668153:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985388800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320296.668202:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389184 00000100:00100000:2.0:1767320296.668206:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389184 00000100:00100000:2.0:1767320296.668235:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389056 00000100:00100000:2.0:1767320296.668238:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389056 00000100:00100000:2.0:1767320296.668250:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5137 00000100:00100000:2.0:1767320296.668257:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985389184:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320296.668296:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985389184:12345-0@lo:400:kworker.0 Request processed in 39us (239us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320296.668302:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5137 00000100:00100000:2.0:1767320296.668313:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5786 00000100:00100000:2.0:1767320296.668318:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985388672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320296.668328:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320296.668355:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985388672:12345-0@lo:38:kworker.0 Request processed in 35us (711us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320296.668361:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5786 00000100:00080000:0.0:1767320296.668376:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767320296.668377:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5787 00000100:00080000:0.0:1767320296.668383:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1767320296.668385:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985389056:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767320296.668386:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320296.668395:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5539 00000100:00080000:0.0:1767320296.668398:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00010000:00020000:2.0:1767320296.668401:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320296.668404:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320296.668423:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c700 x1853163985389312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320296.668430:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985389056:12345-0@lo:38:kworker.0 Request processed in 42us (309us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320296.668440:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5787 00000100:00080000:2.0:1767320296.668456:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320296.668478:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985389184:0@lo:400:kworker.0 00000100:00100000:0.0:1767320296.668497:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985389568:0@lo:400:kworker.0 00000100:00100000:0.0:1767320296.668540:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320296.668561:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320296.668567:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985389440:0@lo:400:kworker.0 00000100:00100000:0.0:1767320296.668589:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320296.668614:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389568 00000100:00100000:0.0:1767320296.668618:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389568 00000100:00100000:0.0:1767320296.668651:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389440 00000100:00100000:0.0:1767320296.668654:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389440 00000100:00100000:0.0:1767320296.668672:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3343 00000100:00100000:0.0:1767320296.668678:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985389568:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320296.668709:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985389568:12345-0@lo:400:kworker.0 Request processed in 31us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320296.668715:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3343 00000100:00100000:0.0:1767320296.668727:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3344 00000100:00100000:0.0:1767320296.668733:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985389440:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320296.668756:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985389440:12345-0@lo:400:kworker.0 Request processed in 24us (167us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320296.668761:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3344 00000100:00080000:0.0:1767320296.668781:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320296.668784:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320296.668797:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985389440:0@lo:400:kworker.0 00000100:00100000:0.0:1767320296.668805:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985389568:0@lo:400:kworker.0 00000100:00100000:2.0:1767320296.669289:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985389312:0@lo:38:kworker.0 00000100:00100000:2.0:1767320296.669323:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320296.669396:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320296.669401:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320296.669404:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985389056:0@lo:38:kworker.0 00000100:00080000:2.0:1767320296.669413:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320296.669416:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320296.669418:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985388672:0@lo:38:kworker.0 00000100:00100000:0.0:1767320296.674060:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389312 00000100:00100000:0.0:1767320296.674077:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389312 00000100:00100000:0.0:1767320296.674132:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5788 00000100:00100000:0.0:1767320296.674140:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985389312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320296.674177:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320296.674259:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985389312:12345-0@lo:38:kworker.0 Request processed in 119us (4935us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320296.674270:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5788 00000100:00080000:2.0:1767320296.674470:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320296.674475:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320296.674480:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985389312:0@lo:38:kworker.0 00000100:00100000:1.0:1767320296.684713:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985388928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320296.684854:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985388928:12345-0@lo:400:kworker.0 Request processed in 16897us (17236us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320296.684868:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5136 00010000:00010000:1.0:1767320297.314362:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320298.203633:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985389696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320298.203715:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320298.203889:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320298.203905:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985389824:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320298.203934:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320298.204007:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389696 00000100:00100000:2.0:1767320298.204021:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389696 00000100:00100000:2.0:1767320298.204153:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389824 00000100:00100000:2.0:1767320298.204158:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389824 00000100:00100000:3.0:1767320298.204253:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2540 00000100:00100000:3.0:1767320298.204266:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985389696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320298.204310:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320298.204390:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985389696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 126us (674us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320298.204398:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2540 00000100:00100000:3.0:1767320298.204473:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985389696:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320298.208302:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320298.208320:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320298.208882:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2541 00000100:00100000:2.0:1767320298.208906:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985389824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320298.208954:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320298.209071:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985389824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 177us (5134us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320298.209080:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2541 00000100:00100000:3.0:1767320298.210840:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985389824:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320298.331543:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320299.364385:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320300.379499:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320300.715916:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320300.716056:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609408 00000100:00100000:0.0:1767320300.716067:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609408 00000100:00100000:0.0:1767320300.716255:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5138 00000100:00100000:0.0:1767320300.716265:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920609408:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320300.716347:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920609408:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (440us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320300.716354:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5138 00000100:00080000:1.0:1767320300.716579:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320300.718032:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320300.718206:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609280 00000100:00100000:0.0:1767320300.718215:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609280 00000100:00100000:0.0:1767320300.718259:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5789 00000100:00100000:0.0:1767320300.718267:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920609280:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320300.718289:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320300.718351:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920609280:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 82us (324us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320300.718358:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5789 00010000:00010000:2.0:1767320301.405192:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320301.787830:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320301.787841:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320301.787845:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320301.787850:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320301.787854:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320301.787864:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5544 00000100:00080000:2.0:1767320301.787869:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320301.787876:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320301.787917:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985389952/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320301.787946:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320301.787949:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320301.787951:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320301.787954:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5544 00000100:00080000:2.0:1767320301.787957:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320301.787960:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320301.787972:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad500 x1853163985390080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320301.788014:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320301.788018:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320301.788019:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320301.788023:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5544 00000100:00080000:2.0:1767320301.788025:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320301.788029:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320301.788038:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adc00 x1853163985390336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320301.788051:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320301.788054:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320301.788056:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320301.788059:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5544 00000100:00080000:2.0:1767320301.788061:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320301.788064:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320301.788074:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adf80 x1853163985390592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320301.788133:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985390720:0@lo:400:kworker.0 00000100:00100000:2.0:1767320301.788208:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320301.788263:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320301.788268:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985390848:0@lo:400:kworker.0 00000100:00100000:2.0:1767320301.788298:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320301.788377:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985390464:0@lo:400:kworker.0 00000100:00100000:1.0:1767320301.788407:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320301.788434:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320301.788439:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985390208:0@lo:400:kworker.0 00000100:00100000:1.0:1767320301.788456:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320301.788724:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390464 00000100:00100000:2.0:1767320301.788732:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390464 00000100:00100000:2.0:1767320301.788839:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390208 00000100:00100000:2.0:1767320301.788843:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390208 00000100:00100000:0.0:1767320301.788949:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390720 00000100:00100000:0.0:1767320301.788953:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390720 00000100:00100000:2.0:1767320301.792635:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5139 00000100:00100000:2.0:1767320301.792651:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985390464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320301.792761:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985390464:12345-0@lo:400:kworker.0 Request processed in 102us (4340us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320301.792768:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5139 00000100:00100000:2.0:1767320301.792787:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5140 00000100:00100000:2.0:1767320301.792794:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985390208:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320301.792824:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985390208:12345-0@lo:400:kworker.0 Request processed in 31us (4367us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320301.792830:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5140 00000100:00100000:2.0:1767320301.792881:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390848 00000100:00100000:2.0:1767320301.792889:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390848 00000100:00100000:2.0:1767320301.792927:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3345 00000100:00100000:2.0:1767320301.792933:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985390848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320301.792969:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985390848:12345-0@lo:400:kworker.0 Request processed in 36us (4670us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320301.792974:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3345 00000100:00100000:2.0:1767320301.793000:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985389952:0@lo:38:kworker.0 00000100:00100000:2.0:1767320301.793038:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320301.793071:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985390080:0@lo:38:kworker.0 00000100:00100000:2.0:1767320301.793097:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320301.793133:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985390336:0@lo:38:kworker.0 00000100:00100000:2.0:1767320301.793196:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320301.793215:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985390592:0@lo:38:kworker.0 00000100:00100000:2.0:1767320301.793236:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320301.793268:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390080 00000100:00100000:2.0:1767320301.793272:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390080 00000100:00100000:2.0:1767320301.793304:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1938 00000100:00100000:2.0:1767320301.793309:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985390080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320301.793329:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320301.793468:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985389952 00000100:00100000:3.0:1767320301.793472:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985389952 00000100:00100000:3.0:1767320301.793495:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390336 00000100:00100000:3.0:1767320301.793498:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390336 00000100:00100000:3.0:1767320301.793506:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390592 00000100:00100000:3.0:1767320301.793509:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390592 00000100:00100000:3.0:1767320301.793518:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5790 00000100:00100000:3.0:1767320301.793522:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985389952:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320301.793533:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320301.793566:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985389952:12345-0@lo:38:kworker.0 Request processed in 43us (528us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320301.793572:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5790 00000100:00100000:3.0:1767320301.793583:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5791 00000100:00100000:3.0:1767320301.793587:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985390336:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320301.793595:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320301.793615:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985390336:12345-0@lo:38:kworker.0 Request processed in 27us (420us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320301.793620:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5791 00000100:00100000:3.0:1767320301.793629:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5792 00000100:00100000:3.0:1767320301.793633:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985390592:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320301.793641:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320301.793661:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985390592:12345-0@lo:38:kworker.0 Request processed in 26us (423us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320301.793665:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5792 00000100:00080000:3.0:1767320301.793700:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320301.793704:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320301.793708:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985389952:0@lo:38:kworker.0 00000100:00080000:3.0:1767320301.793717:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320301.793721:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320301.793724:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985390336:0@lo:38:kworker.0 00000100:00080000:3.0:1767320301.793731:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320301.793734:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320301.793737:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985390592:0@lo:38:kworker.0 00000100:00100000:3.0:1767320301.793802:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985390848:0@lo:400:kworker.0 00000100:00100000:3.0:1767320301.793870:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985390208:0@lo:400:kworker.0 00000100:00100000:3.0:1767320301.793881:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985390464:0@lo:400:kworker.0 00000100:00080000:1.0:1767320301.794183:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320301.794188:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320301.794191:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320301.794488:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3346 00000100:00100000:0.0:1767320301.794497:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985390720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320301.794542:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985390720:12345-0@lo:400:kworker.0 Request processed in 47us (6333us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320301.794547:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3346 00000100:00100000:0.0:1767320301.794573:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985390720:0@lo:400:kworker.0 00000100:00100000:2.0:1767320301.794764:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985390080:12345-0@lo:38:kworker.0 Request processed in 1453us (1666us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320301.794771:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1938 00000100:00080000:3.0:1767320301.796759:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320301.796771:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320301.796776:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985390080:0@lo:38:kworker.0 00010000:00010000:1.0:1767320302.431343:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320303.323854:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1767320303.323880:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985390976:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320303.323964:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320303.324009:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985391104:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320303.324042:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320303.324254:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985390976 00000100:00100000:0.0:1767320303.324262:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985390976 00000100:00100000:2.0:1767320303.324314:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985391104 00000100:00100000:2.0:1767320303.324320:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985391104 00000100:00100000:0.0:1767320303.324328:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2542 00000100:00100000:0.0:1767320303.324337:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985390976:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320303.324346:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2543 00000100:00100000:2.0:1767320303.324351:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985391104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320303.324367:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:2.0:1767320303.324367:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767320303.324419:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320303.324420:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985391104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (377us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320303.324423:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320303.324428:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2543 00000100:00100000:1.0:1767320303.324455:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985391104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320303.329315:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985390976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4968us (5340us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320303.329329:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2542 00000100:00100000:1.0:1767320303.333552:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985390976:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320303.451532:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320304.475582:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320305.499424:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320305.834757:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320305.834867:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320305.839811:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609664 00000100:00100000:0.0:1767320305.839827:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609664 00000100:00100000:0.0:1767320305.839913:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609536 00000100:00100000:0.0:1767320305.839917:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609536 00000100:00100000:0.0:1767320305.839948:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5141 00000100:00100000:0.0:1767320305.839958:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920609664:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320305.840055:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920609664:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 98us (5308us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320305.840063:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5141 00000100:00100000:0.0:1767320305.840083:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5793 00000100:00100000:0.0:1767320305.840087:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920609536:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320305.840108:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320305.840133:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920609536:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (5266us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320305.840139:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5793 00000100:00080000:2.0:1767320305.843533:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320306.527289:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320306.907289:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320306.907301:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320306.907304:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320306.907308:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320306.907313:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320306.907321:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5550 00000100:00080000:2.0:1767320306.907326:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320306.907332:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320306.907371:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63b80 x1853163985391232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320306.907397:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320306.907401:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320306.907403:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320306.907406:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5550 00000100:00080000:2.0:1767320306.907410:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320306.907413:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320306.907425:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61c00 x1853163985391360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320306.907468:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320306.907472:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320306.907474:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320306.907478:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5550 00000100:00080000:2.0:1767320306.907480:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320306.907483:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320306.907493:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61180 x1853163985391616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320306.907518:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320306.907521:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320306.907523:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320306.907526:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5550 00000100:00080000:2.0:1767320306.907528:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320306.907531:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320306.907538:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163985391872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320306.907573:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985391744:0@lo:400:kworker.0 00000100:00100000:2.0:1767320306.907642:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320306.907714:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320306.907719:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985392000:0@lo:400:kworker.0 00000100:00100000:2.0:1767320306.907745:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320306.908459:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985391488:0@lo:400:kworker.0 00000100:00100000:1.0:1767320306.908496:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320306.908811:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985392128:0@lo:400:kworker.0 00000100:00100000:0.0:1767320306.912707:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985391232:0@lo:38:kworker.0 00000100:00100000:3.0:1767320306.914866:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320306.914954:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392000 00000100:00100000:3.0:1767320306.914962:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392000 00000100:00100000:3.0:1767320306.915020:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392128 00000100:00100000:3.0:1767320306.915023:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392128 00000100:00100000:3.0:1767320306.915043:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3347 00000100:00100000:3.0:1767320306.915051:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985392000:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320306.915108:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985392000:12345-0@lo:400:kworker.0 Request processed in 57us (7361us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320306.915116:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3347 00000100:00100000:3.0:1767320306.915923:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3348 00000100:00100000:3.0:1767320306.915936:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985392128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320306.915994:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985392128:12345-0@lo:400:kworker.0 Request processed in 64us (1140us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320306.916001:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3348 00000100:00080000:3.0:1767320306.916024:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320306.916028:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320306.916061:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985392128:0@lo:400:kworker.0 00000100:00100000:3.0:1767320306.916092:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985392000:0@lo:400:kworker.0 00000100:00100000:3.0:1767320306.916301:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985391744 00000100:00100000:3.0:1767320306.916307:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985391744 00000100:00080000:3.0:1767320306.916326:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320306.916357:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985391488 00000100:00100000:3.0:1767320306.916361:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985391488 00000100:00100000:3.0:1767320306.916389:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5142 00000100:00100000:3.0:1767320306.916395:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985391744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320306.916434:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985391744:12345-0@lo:400:kworker.0 Request processed in 38us (8791us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320306.916440:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5142 00000100:00100000:3.0:1767320306.916451:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5143 00000100:00100000:3.0:1767320306.916457:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985391488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320306.916483:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985391488:12345-0@lo:400:kworker.0 Request processed in 27us (7986us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320306.916488:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5143 00000100:00080000:3.0:1767320306.916500:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320306.916515:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985391744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320306.916766:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985391488:0@lo:400:kworker.0 00000100:00100000:0.0:1767320306.925713:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320306.925892:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985391360:0@lo:38:kworker.0 00000100:00100000:0.0:1767320306.925944:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320306.925977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985391616:0@lo:38:kworker.0 00000100:00100000:1.0:1767320306.925984:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985391360 00000100:00100000:1.0:1767320306.925995:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985391360 00000100:00100000:0.0:1767320306.926004:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320306.926038:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1939 00000100:00100000:1.0:1767320306.926046:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985391360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320306.926066:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320306.926112:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985391360:12345-0@lo:38:kworker.0 Request processed in 65us (167us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320306.926119:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1939 00000100:00100000:1.0:1767320306.934337:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985391872:0@lo:38:kworker.0 00000100:00100000:1.0:1767320306.934393:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320306.936358:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985391232 00000100:00100000:2.0:1767320306.936373:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985391232 00000100:00100000:2.0:1767320306.936439:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985391616 00000100:00100000:2.0:1767320306.936443:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985391616 00000100:00100000:2.0:1767320306.936478:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5794 00000100:00100000:2.0:1767320306.936488:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985391232:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320306.936739:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5795 00000100:00100000:3.0:1767320306.936746:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985391616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320306.937297:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320306.937370:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985391232:12345-0@lo:38:kworker.0 Request processed in 882us (12556us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320306.937378:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5794 00000100:00080000:1.0:1767320306.937663:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320306.937671:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320306.937675:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985391232:0@lo:38:kworker.0 00000100:00080000:1.0:1767320306.937689:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320306.937692:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320306.937695:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985391360:0@lo:38:kworker.0 00010000:00020000:3.0:1767320306.938443:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320306.938555:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985391616:12345-0@lo:38:kworker.0 Request processed in 1806us (12548us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320306.938564:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5795 00000100:00100000:3.0:1767320306.938587:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985391872 00000100:00100000:3.0:1767320306.938593:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985391872 00000100:00100000:2.0:1767320306.941071:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5796 00000100:00080000:1.0:1767320306.941077:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767320306.941084:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985391872:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767320306.941088:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320306.941097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985391616:0@lo:38:kworker.0 00010000:00020000:2.0:1767320306.941115:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320306.941204:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985391872:12345-0@lo:38:kworker.0 Request processed in 121us (6809us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320306.941212:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5796 00000100:00080000:1.0:1767320306.941305:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320306.941310:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320306.941313:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985391872:0@lo:38:kworker.0 00010000:00010000:2.0:1767320307.547981:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320308.443470:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985392256:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320308.443541:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320308.443658:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392256 00000100:00100000:2.0:1767320308.443666:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392256 00000100:00100000:2.0:1767320308.443735:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2544 00000100:00100000:2.0:1767320308.443746:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985392256:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320308.443776:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320308.443823:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985392256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (282us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320308.443831:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2544 00000100:00100000:2.0:1767320308.443873:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985392256:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320308.444315:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320308.444448:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985392384:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320308.444476:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320308.444555:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392384 00000100:00100000:0.0:1767320308.444560:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392384 00000100:00100000:0.0:1767320308.444597:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2545 00000100:00100000:0.0:1767320308.444604:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985392384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320308.444619:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320308.444665:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985392384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (187us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320308.444672:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2545 00000100:00100000:0.0:1767320308.444700:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985392384:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320308.444754:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320308.578940:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320309.598032:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320310.620129:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320310.956235:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320310.956295:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320310.957067:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609792 00000100:00100000:0.0:1767320310.957084:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609792 00000100:00100000:0.0:1767320310.957146:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920609920 00000100:00100000:0.0:1767320310.957150:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920609920 00000100:00100000:3.0:1767320310.957216:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5797 00000100:00100000:3.0:1767320310.957229:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920609792:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320310.957260:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320310.957270:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5144 00000100:00100000:0.0:1767320310.957284:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920609920:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320310.957365:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920609792:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 133us (1194us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320310.957377:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5797 00000100:00100000:0.0:1767320310.957386:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920609920:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 97us (1082us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320310.957400:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5144 00000100:00080000:1.0:1767320310.959600:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320311.645108:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320312.029782:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320312.029792:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320312.029795:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320312.029800:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320312.029805:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320312.029813:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5555 00000100:00080000:2.0:1767320312.029817:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320312.029825:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320312.029864:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62300 x1853163985392512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320312.029896:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320312.029900:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320312.029901:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320312.029905:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5555 00000100:00080000:2.0:1767320312.029907:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320312.029910:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320312.029923:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62d80 x1853163985392640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320312.029972:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320312.029977:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320312.029979:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320312.029982:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5555 00000100:00080000:2.0:1767320312.029985:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320312.029988:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320312.030001:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163985392896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320312.030027:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320312.030030:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320312.030032:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320312.030035:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5555 00000100:00080000:2.0:1767320312.030037:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320312.030039:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320312.030049:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61180 x1853163985393152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320312.030089:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985393280:0@lo:400:kworker.0 00000100:00100000:2.0:1767320312.030154:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320312.030238:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320312.030244:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985393024:0@lo:400:kworker.0 00000100:00100000:2.0:1767320312.030268:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320312.030304:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985393024 00000100:00100000:2.0:1767320312.030311:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985393024 00000100:00100000:2.0:1767320312.030362:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5145 00000100:00100000:2.0:1767320312.030372:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985393024:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320312.030417:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985393024:12345-0@lo:400:kworker.0 Request processed in 47us (148us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320312.030425:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5145 00000100:00100000:2.0:1767320312.030466:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985393024:0@lo:400:kworker.0 00000100:00100000:0.0:1767320312.030653:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985393408:0@lo:400:kworker.0 00000100:00100000:0.0:1767320312.030680:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320312.030709:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320312.030714:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985392768:0@lo:400:kworker.0 00000100:00100000:0.0:1767320312.030733:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320312.030756:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392768 00000100:00100000:0.0:1767320312.030759:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392768 00000100:00100000:0.0:1767320312.030784:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5146 00000100:00100000:0.0:1767320312.030789:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985392768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320312.030818:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985392768:12345-0@lo:400:kworker.0 Request processed in 28us (84us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320312.030823:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5146 00000100:00100000:0.0:1767320312.030879:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985392768:0@lo:400:kworker.0 00000100:00100000:2.0:1767320312.031249:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985393280 00000100:00100000:2.0:1767320312.031254:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985393280 00000100:00100000:2.0:1767320312.031291:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985393408 00000100:00100000:2.0:1767320312.031294:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985393408 00000100:00100000:2.0:1767320312.031314:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3349 00000100:00100000:2.0:1767320312.031320:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985393280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320312.031361:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985393280:12345-0@lo:400:kworker.0 Request processed in 42us (1207us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320312.031368:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3349 00000100:00100000:2.0:1767320312.031379:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3350 00000100:00100000:2.0:1767320312.031384:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985393408:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320312.031406:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985393408:12345-0@lo:400:kworker.0 Request processed in 23us (725us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320312.031411:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3350 00000100:00100000:2.0:1767320312.031437:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985393280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320312.031586:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985393408:0@lo:400:kworker.0 00000100:00100000:1.0:1767320312.031718:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985392512:0@lo:38:kworker.0 00000100:00100000:1.0:1767320312.031755:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320312.031801:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985392640:0@lo:38:kworker.0 00000100:00100000:1.0:1767320312.031828:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320312.031857:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985392896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320312.031875:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320312.031894:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985393152:0@lo:38:kworker.0 00000100:00100000:1.0:1767320312.031911:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320312.031953:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320312.031959:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320312.031963:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320312.032131:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392640 00000100:00100000:3.0:1767320312.032146:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392640 00000100:00100000:0.0:1767320312.032147:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392512 00000100:00100000:0.0:1767320312.032155:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392512 00000100:00100000:0.0:1767320312.032219:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985392896 00000100:00100000:0.0:1767320312.032223:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985392896 00000100:00100000:2.0:1767320312.032300:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985393152 00000100:00100000:2.0:1767320312.032305:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985393152 00000100:00100000:2.0:1767320312.034890:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5798 00000100:00100000:2.0:1767320312.034903:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985392512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320312.034926:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320312.035026:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985392512:12345-0@lo:38:kworker.0 Request processed in 123us (3268us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320312.035032:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5798 00000100:00100000:2.0:1767320312.035051:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5799 00000100:00100000:2.0:1767320312.035056:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985392896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320312.035066:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320312.035089:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985392896:12345-0@lo:38:kworker.0 Request processed in 32us (3212us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320312.035094:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5799 00000100:00100000:2.0:1767320312.035106:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5800 00000100:00100000:2.0:1767320312.035110:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985393152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320312.035120:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320312.035142:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985393152:12345-0@lo:38:kworker.0 Request processed in 30us (3230us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320312.035148:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5800 00000100:00080000:2.0:1767320312.035206:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320312.035211:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320312.035215:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985392512:0@lo:38:kworker.0 00000100:00080000:2.0:1767320312.035225:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320312.035228:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320312.035231:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985392896:0@lo:38:kworker.0 00000100:00080000:2.0:1767320312.035238:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320312.035241:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320312.035244:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985393152:0@lo:38:kworker.0 00000100:00100000:3.0:1767320312.036557:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1940 00000100:00100000:3.0:1767320312.036581:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985392640:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320312.036608:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320312.036714:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985392640:12345-0@lo:38:kworker.0 Request processed in 133us (4884us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320312.036722:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1940 00000100:00080000:3.0:1767320312.036960:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320312.036966:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320312.036971:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985392640:0@lo:38:kworker.0 00010000:00010000:0.0:1767320312.667638:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320313.563546:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320313.563578:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985393536:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320313.563684:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320313.563795:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985393536 00000100:00100000:0.0:1767320313.563805:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985393536 00000100:00100000:0.0:1767320313.563871:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2546 00000100:00100000:0.0:1767320313.563883:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985393536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320313.563921:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320313.563973:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985393536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320313.563994:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2546 00000100:00100000:3.0:1767320313.564106:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985393536:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320313.564865:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985393664:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320313.564920:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320313.565000:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985393664 00000100:00100000:1.0:1767320313.565008:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985393664 00000100:00100000:1.0:1767320313.565058:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2547 00000100:00100000:1.0:1767320313.565068:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985393664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320313.565096:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320313.565543:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985393664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 466us (611us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320313.565553:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2547 00000100:00100000:3.0:1767320313.565728:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985393664:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320313.699451:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320314.717330:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320315.741682:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320316.076139:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320316.076503:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767320316.076582:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920610304 offset 0 length 520: 4 00000100:00100000:1.0:1767320316.076683:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920610176 00000100:00100000:1.0:1767320316.076697:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920610176 00000100:00100000:1.0:1767320316.076797:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920610048 00000100:00100000:1.0:1767320316.076801:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920610048 00000100:00100000:1.0:1767320316.076828:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5147 00000100:00100000:1.0:1767320316.076850:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920610176:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320316.076951:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920610176:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 104us (825us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320316.076959:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5147 00000100:00100000:1.0:1767320316.076981:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5801 00000100:00100000:1.0:1767320316.076985:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920610048:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320316.077009:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320316.077037:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920610048:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 50us (536us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320316.077043:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5801 00000100:00080000:3.0:1767320316.077679:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320316.124104:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320291/real 1767320291] req@ffff8ce1af73f800 x1853163985387264/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320316 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320316.124341:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1af73f800 x1853163985387264/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320316 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320316.124370:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320316.124403:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320316.124408:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985387264:0@lo:250:kworker.0 00000100:00080000:0.0:1767320316.124552:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320316.124559:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320316.124562:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320316.124568:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5539 00000100:00080000:0.0:1767320316.124572:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320316.124579:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320316.124624:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985393792/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320316.124649:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320316.124653:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320316.124655:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320316.124659:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5560 00000100:00080000:0.0:1767320316.124662:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320316.124665:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320316.124682:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ea00 x1853163985393920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320316.124692:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320316.124695:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320316.124697:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320316.124700:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5560 00000100:00080000:0.0:1767320316.124703:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320316.124707:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320316.124721:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ca80 x1853163985394048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320316.124793:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320316.124797:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320316.124799:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320316.124803:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5560 00000100:00080000:0.0:1767320316.124805:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320316.124809:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320316.124825:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ed80 x1853163985394304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320316.124869:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320316.124872:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320316.124874:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320316.124878:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5560 00000100:00080000:0.0:1767320316.124881:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320316.124884:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320316.124898:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18f100 x1853163985394560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320316.124944:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985394688:0@lo:400:kworker.0 00000100:00100000:0.0:1767320316.125028:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320316.125105:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320316.125121:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985394432:0@lo:400:kworker.0 00000100:00100000:0.0:1767320316.125150:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320316.125765:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985394816:0@lo:400:kworker.0 00000100:00100000:1.0:1767320316.125815:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320316.125856:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320316.125862:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985394176:0@lo:400:kworker.0 00000100:00100000:1.0:1767320316.125889:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320316.125926:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394432 00000100:00100000:1.0:1767320316.125935:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394432 00000100:00100000:1.0:1767320316.125997:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394176 00000100:00100000:1.0:1767320316.126000:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394176 00000100:00100000:1.0:1767320316.126029:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5148 00000100:00100000:0.0:1767320316.126528:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394688 00000100:00100000:0.0:1767320316.126534:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394688 00000100:00100000:0.0:1767320316.126572:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394816 00000100:00100000:0.0:1767320316.126576:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394816 00000100:00100000:0.0:1767320316.126597:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3351 00000100:00100000:0.0:1767320316.126606:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985394688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320316.126681:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985394688:12345-0@lo:400:kworker.0 Request processed in 76us (1652us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320316.126689:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3351 00000100:00100000:0.0:1767320316.126708:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3352 00000100:00100000:0.0:1767320316.126714:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985394816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320316.126748:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985394816:12345-0@lo:400:kworker.0 Request processed in 34us (933us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320316.126754:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3352 00000100:00100000:0.0:1767320316.126816:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985394688:0@lo:400:kworker.0 00000100:00080000:3.0:1767320316.127024:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320316.127029:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320316.127032:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320316.127679:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985394432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320316.127750:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985394432:12345-0@lo:400:kworker.0 Request processed in 84us (2599us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320316.127849:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985393792:0@lo:250:kworker.0 00000400:00000100:2.0:1767320316.127885:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985393792 offset 0 length 520: 4 00000100:00100000:2.0:1767320316.127915:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985393920:0@lo:38:kworker.0 00000100:00100000:2.0:1767320316.127942:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320316.127962:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985394048:0@lo:38:kworker.0 00000100:00100000:2.0:1767320316.127985:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320316.128001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985394304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320316.128020:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320316.128050:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985394560:0@lo:38:kworker.0 00000100:00100000:2.0:1767320316.128073:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320316.135092:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5149 00000100:00100000:0.0:1767320316.135306:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985394176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320316.135440:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985394176:12345-0@lo:400:kworker.0 Request processed in 331us (9549us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320316.135451:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5149 00000100:00100000:1.0:1767320316.135891:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5148 00000100:00100000:1.0:1767320316.135934:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985393920 00000100:00100000:1.0:1767320316.135955:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985393920 00000100:00100000:3.0:1767320316.136043:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394304 00000100:00100000:3.0:1767320316.136048:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394304 00000100:00100000:3.0:1767320316.136067:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394560 00000100:00100000:3.0:1767320316.136070:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394560 00000100:00100000:3.0:1767320316.136083:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5802 00000100:00100000:3.0:1767320316.136090:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985393920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320316.136126:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320316.136812:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5803 00000100:00100000:0.0:1767320316.136819:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985394304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320316.136833:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320316.136918:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5804 00000100:00100000:2.0:1767320316.136925:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985394560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320316.136937:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320316.137016:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985393920:12345-0@lo:38:kworker.0 Request processed in 925us (9072us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320316.137024:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5802 00000100:00100000:0.0:1767320316.142347:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985394304:12345-0@lo:38:kworker.0 Request processed in 5512us (14310us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320316.142367:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5803 00000100:00100000:2.0:1767320316.142541:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985394560:12345-0@lo:38:kworker.0 Request processed in 5611us (14463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320316.142549:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5804 00000100:00080000:3.0:1767320316.142633:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320316.142639:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320316.142644:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985393920:0@lo:38:kworker.0 00000100:00080000:3.0:1767320316.142666:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320316.142669:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320316.142674:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985394304:0@lo:38:kworker.0 00000100:00080000:3.0:1767320316.142686:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320316.142690:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320316.142693:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985394560:0@lo:38:kworker.0 00000100:00100000:3.0:1767320316.142902:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394048 00000100:00100000:3.0:1767320316.142910:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394048 00000100:00100000:3.0:1767320316.142973:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1941 00000100:00100000:3.0:1767320316.142983:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985394048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320316.143011:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320316.143088:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985394048:12345-0@lo:38:kworker.0 Request processed in 105us (15102us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320316.143094:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1941 00000100:00080000:3.0:1767320316.143131:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320316.143136:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320316.143139:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985394048:0@lo:38:kworker.0 00000100:00100000:0.0:1767320316.147659:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985394432:0@lo:400:kworker.0 00000100:00100000:0.0:1767320316.147729:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985394176:0@lo:400:kworker.0 00000100:00100000:0.0:1767320316.147748:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985394816:0@lo:400:kworker.0 00010000:00010000:2.0:1767320316.763434:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320317.789150:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320318.684699:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985394944:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320318.684797:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320318.684983:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985394944 00000100:00100000:1.0:1767320318.684995:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985394944 00000100:00100000:1.0:1767320318.685093:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2548 00000100:00100000:1.0:1767320318.685106:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985394944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320318.685144:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320318.685235:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985394944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 132us (438us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320318.685247:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2548 00000100:00080000:3.0:1767320318.685525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320318.685616:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985394944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320318.685822:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985395072:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320318.685870:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320318.686007:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395072 00000100:00100000:0.0:1767320318.686014:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395072 00000100:00100000:0.0:1767320318.686062:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2549 00000100:00100000:0.0:1767320318.686071:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985395072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320318.686093:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320318.686134:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985395072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (263us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320318.686141:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2549 00000100:00100000:0.0:1767320318.686202:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985395072:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320318.686360:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767320318.811567:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320319.835897:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320320.859687:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320321.195561:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320321.195681:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320321.197239:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920610560 00000100:00100000:0.0:1767320321.197251:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920610560 00000100:00100000:0.0:1767320321.197360:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920610432 00000100:00100000:0.0:1767320321.197364:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920610432 00000100:00100000:1.0:1767320321.197533:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5150 00000100:00100000:1.0:1767320321.197548:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920610560:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320321.197626:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920610560:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (2075us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320321.197634:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5150 00000100:00100000:0.0:1767320321.198998:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5805 00000100:00100000:0.0:1767320321.199007:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920610432:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320321.199029:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320321.199262:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920610432:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 253us (3580us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320321.199270:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5805 00000100:00080000:3.0:1767320321.200415:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320321.245254:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320321.245264:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320321.245267:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320321.245271:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320321.245276:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320321.245292:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5564 00000100:00080000:3.0:1767320321.245296:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320321.245306:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320321.245350:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adf80 x1853163985395200/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320321.245375:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320321.245378:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320321.245380:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320321.245384:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5564 00000100:00080000:3.0:1767320321.245386:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320321.245389:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320321.245411:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612e300 x1853163985395328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320321.245447:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320321.245451:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320321.245453:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320321.245456:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5564 00000100:00080000:3.0:1767320321.245458:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320321.245461:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320321.245471:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ca80 x1853163985395584/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320321.245492:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320321.245495:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320321.245497:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320321.245501:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5564 00000100:00080000:3.0:1767320321.245503:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320321.245506:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320321.245515:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ea00 x1853163985395840/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320321.245569:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985395456:0@lo:400:kworker.0 00000100:00100000:3.0:1767320321.245579:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985395968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320321.245672:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985396096:0@lo:400:kworker.0 00000100:00100000:3.0:1767320321.248038:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320321.248112:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320321.248122:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985395712:0@lo:400:kworker.0 00000100:00100000:3.0:1767320321.248146:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320321.248197:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395712 00000100:00100000:3.0:1767320321.248204:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395712 00000100:00100000:3.0:1767320321.248253:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5151 00000100:00100000:3.0:1767320321.248260:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985395712:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320321.248304:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985395712:12345-0@lo:400:kworker.0 Request processed in 44us (156us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320321.248310:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5151 00000100:00080000:3.0:1767320321.248328:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320321.248353:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985395712:0@lo:400:kworker.0 00000100:00100000:2.0:1767320321.248380:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320321.248387:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395968 00000100:00100000:3.0:1767320321.248401:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395968 00000100:00100000:3.0:1767320321.248438:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3353 00000100:00100000:3.0:1767320321.248445:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985395968:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320321.248475:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985395968:12345-0@lo:400:kworker.0 Request processed in 31us (444us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320321.248480:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3353 00000100:00100000:3.0:1767320321.248503:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985395968:0@lo:400:kworker.0 00000100:00080000:2.0:1767320321.248520:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320321.248528:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395456 00000100:00100000:3.0:1767320321.248532:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395456 00000100:00100000:3.0:1767320321.248559:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5152 00000100:00100000:3.0:1767320321.248564:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985395456:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320321.248565:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320321.248606:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985395456:12345-0@lo:400:kworker.0 Request processed in 42us (227us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320321.248612:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5152 00000100:00100000:2.0:1767320321.248926:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985395456:0@lo:400:kworker.0 00000100:00100000:0.0:1767320321.249121:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985395200:0@lo:38:kworker.0 00000100:00100000:0.0:1767320321.249156:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320321.249209:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985395328:0@lo:38:kworker.0 00000100:00100000:0.0:1767320321.249234:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320321.249245:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395200 00000100:00100000:2.0:1767320321.249251:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395200 00000100:00100000:0.0:1767320321.249260:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985395584:0@lo:38:kworker.0 00000100:00100000:2.0:1767320321.249279:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5806 00000100:00100000:2.0:1767320321.249286:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985395200:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320321.249292:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767320321.249305:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320321.249312:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985395840:0@lo:38:kworker.0 00000100:00100000:0.0:1767320321.249342:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320321.249350:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985395200:12345-0@lo:38:kworker.0 Request processed in 62us (191us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320321.249358:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5806 00000100:00100000:2.0:1767320321.249370:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395584 00000100:00100000:2.0:1767320321.249374:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395584 00000100:00080000:0.0:1767320321.249378:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320321.249384:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320321.249389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985395200:0@lo:38:kworker.0 00000100:00100000:2.0:1767320321.249403:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395840 00000100:00100000:2.0:1767320321.249407:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395840 00000100:00100000:2.0:1767320321.249422:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5807 00000100:00100000:2.0:1767320321.249428:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985395584:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320321.249430:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985395328 00000100:00100000:0.0:1767320321.249436:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985395328 00010000:00020000:2.0:1767320321.249439:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320321.249459:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1942 00000100:00100000:0.0:1767320321.249465:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985395328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320321.249476:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320321.249484:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985395584:12345-0@lo:38:kworker.0 Request processed in 55us (191us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320321.249495:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5807 00000100:00100000:0.0:1767320321.249512:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985395328:12345-0@lo:38:kworker.0 Request processed in 44us (275us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320321.249512:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5808 00000100:00100000:2.0:1767320321.249519:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985395840:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320321.249525:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1942 00010000:00020000:2.0:1767320321.249531:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320321.249560:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985395840:12345-0@lo:38:kworker.0 Request processed in 42us (217us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320321.249566:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5808 00000100:00080000:0.0:1767320321.249570:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320321.249577:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320321.249580:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985395328:0@lo:38:kworker.0 00000100:00080000:0.0:1767320321.249589:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320321.249593:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320321.249596:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985395584:0@lo:38:kworker.0 00000100:00080000:0.0:1767320321.249602:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320321.249605:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320321.249607:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985395840:0@lo:38:kworker.0 00000100:00100000:1.0:1767320321.253638:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320321.254234:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396096 00000100:00100000:3.0:1767320321.254246:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396096 00000100:00080000:2.0:1767320321.254481:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320321.254558:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3354 00000100:00100000:3.0:1767320321.254569:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985396096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320321.254745:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985396096:12345-0@lo:400:kworker.0 Request processed in 178us (1113us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320321.254752:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3354 00000100:00100000:1.0:1767320321.262261:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985396096:0@lo:400:kworker.0 00010000:00010000:1.0:1767320321.887208:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320322.907466:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320323.803468:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1767320323.803493:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985396224:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320323.803570:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320323.803639:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985396352:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320323.803664:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320323.803687:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396224 00000100:00100000:0.0:1767320323.803695:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396224 00000100:00080000:2.0:1767320323.803731:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320323.803745:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396352 00000100:00100000:0.0:1767320323.803747:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396352 00000100:00100000:2.0:1767320323.803758:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2550 00000100:00100000:2.0:1767320323.803769:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985396224:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320323.803783:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2551 00000100:00100000:0.0:1767320323.803789:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985396352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320323.803798:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:0.0:1767320323.803801:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320323.803843:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985396224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (268us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320323.803850:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2550 00000100:00080000:1.0:1767320323.803862:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320323.809821:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985396352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 6021us (6146us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320323.809843:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2551 00000100:00100000:2.0:1767320323.816810:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985396224:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320323.816846:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985396352:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320323.931894:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320324.955393:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320325.981950:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320326.314350:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320326.314490:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320326.321996:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920610816 00000100:00100000:1.0:1767320326.322017:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920610816 00000100:00100000:1.0:1767320326.322143:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920610688 00000100:00100000:1.0:1767320326.322148:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920610688 00000100:00100000:1.0:1767320326.322198:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5153 00000100:00100000:1.0:1767320326.322209:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920610816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320326.322316:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920610816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (7980us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320326.322323:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5153 00000100:00100000:1.0:1767320326.322341:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5809 00000100:00100000:1.0:1767320326.322344:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920610688:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320326.322364:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320326.322385:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920610688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (7898us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320326.322389:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5809 00000100:00080000:2.0:1767320326.323452:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320326.364598:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320326.364610:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320326.364614:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320326.364618:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320326.364624:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320326.364635:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5569 00000100:00080000:3.0:1767320326.364804:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320326.364817:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320326.364872:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985396480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320326.364894:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320326.364897:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320326.364900:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320326.364904:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5569 00000100:00080000:3.0:1767320326.364907:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320326.364913:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320326.364914:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985396480:0@lo:38:kworker.0 00000100:00100000:3.0:1767320326.364930:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05880 x1853163985396608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320326.364977:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320326.364981:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320326.364983:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320326.364987:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5569 00000100:00080000:3.0:1767320326.364990:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320326.364994:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320326.365012:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320326.365012:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357aa00 x1853163985396864/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320326.365035:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320326.365039:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320326.365041:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320326.365046:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5569 00000100:00080000:3.0:1767320326.365049:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320326.365053:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320326.365066:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ca80 x1853163985397120/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320326.365094:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985396608:0@lo:38:kworker.0 00000100:00100000:2.0:1767320326.365128:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320326.365157:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985396864:0@lo:38:kworker.0 00000100:00100000:2.0:1767320326.365202:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320326.365222:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985397120:0@lo:38:kworker.0 00000100:00100000:2.0:1767320326.365246:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320326.365278:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985397248:0@lo:400:kworker.0 00000100:00100000:2.0:1767320326.365318:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320326.365342:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320326.365348:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985397376:0@lo:400:kworker.0 00000100:00100000:2.0:1767320326.365370:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320326.365831:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985396992:0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.365871:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320326.365896:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320326.365902:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985396736:0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.365950:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320326.365988:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985397248 00000100:00100000:3.0:1767320326.365998:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985397248 00000100:00100000:3.0:1767320326.366062:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985397376 00000100:00100000:3.0:1767320326.366065:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985397376 00000100:00100000:3.0:1767320326.366090:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3355 00000100:00100000:3.0:1767320326.366100:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985397248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.366570:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985397248:12345-0@lo:400:kworker.0 Request processed in 469us (1248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320326.366579:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3355 00000100:00100000:3.0:1767320326.366855:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3356 00000100:00100000:3.0:1767320326.366867:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985397376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.366911:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985397376:12345-0@lo:400:kworker.0 Request processed in 49us (1539us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320326.366917:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3356 00000100:00100000:3.0:1767320326.367044:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396480 00000100:00100000:3.0:1767320326.367051:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396480 00000100:00100000:3.0:1767320326.367078:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396864 00000100:00100000:3.0:1767320326.367082:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396864 00000100:00100000:3.0:1767320326.367089:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985397120 00000100:00100000:3.0:1767320326.367091:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985397120 00000100:00100000:3.0:1767320326.367099:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396992 00000100:00100000:3.0:1767320326.367102:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396992 00000100:00100000:3.0:1767320326.367122:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396736 00000100:00100000:3.0:1767320326.367124:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396736 00000100:00100000:3.0:1767320326.367138:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5154 00000100:00100000:3.0:1767320326.367145:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985396992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.367224:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985396992:12345-0@lo:400:kworker.0 Request processed in 79us (1351us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320326.367230:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5154 00000100:00100000:3.0:1767320326.367243:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5155 00000100:00100000:3.0:1767320326.367249:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985396736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.367279:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985396736:12345-0@lo:400:kworker.0 Request processed in 30us (1329us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320326.367284:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5155 00000100:00100000:3.0:1767320326.367295:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5810 00000100:00100000:3.0:1767320326.367310:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985396480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320326.367334:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320326.367386:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985396480:12345-0@lo:38:kworker.0 Request processed in 74us (2378us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320326.367393:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5810 00000100:00100000:3.0:1767320326.367404:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5811 00000100:00100000:3.0:1767320326.367408:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985396864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320326.367419:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320326.367935:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985396864:12345-0@lo:38:kworker.0 Request processed in 521us (2728us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320326.367943:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5811 00000100:00100000:3.0:1767320326.367960:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5812 00000100:00100000:3.0:1767320326.367964:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985397120:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320326.367977:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320326.368008:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985397120:12345-0@lo:38:kworker.0 Request processed in 42us (2762us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320326.368014:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5812 00000100:00100000:3.0:1767320326.368100:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985396736:0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.368114:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985396992:0@lo:400:kworker.0 00000100:00080000:3.0:1767320326.368147:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320326.368154:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320326.368157:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320326.368215:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985397376:0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.368227:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985397248:0@lo:400:kworker.0 00000100:00080000:3.0:1767320326.368257:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320326.368263:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320326.368266:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985396864:0@lo:38:kworker.0 00000100:00080000:3.0:1767320326.368275:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320326.368279:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320326.368282:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985397120:0@lo:38:kworker.0 00000100:00080000:3.0:1767320326.368292:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320326.368308:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320326.368311:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985396480:0@lo:38:kworker.0 00000100:00100000:1.0:1767320326.371860:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985396608 00000100:00100000:1.0:1767320326.371884:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985396608 00000100:00100000:1.0:1767320326.371962:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1943 00000100:00100000:1.0:1767320326.371972:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985396608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320326.372000:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320326.372112:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985396608:12345-0@lo:38:kworker.0 Request processed in 140us (6983us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320326.372120:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1943 00000100:00080000:1.0:1767320326.372202:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320326.372210:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320326.372215:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985396608:0@lo:38:kworker.0 00010000:00010000:1.0:1767320327.003965:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320328.027821:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320328.923935:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985397632:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320328.924003:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985397504:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320328.924102:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320328.924116:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320328.925948:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985397504 00000100:00100000:0.0:1767320328.925966:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985397504 00000100:00100000:0.0:1767320328.926065:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985397632 00000100:00100000:0.0:1767320328.926068:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985397632 00000100:00100000:0.0:1767320328.926101:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2552 00000100:00100000:0.0:1767320328.926112:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb3450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985397504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320328.926156:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320328.926276:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb3450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985397504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 165us (2174us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320328.926284:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2552 00000100:00100000:0.0:1767320328.927045:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2553 00000100:00100000:0.0:1767320328.927057:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985397632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320328.927078:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320328.927129:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985397632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (3013us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320328.927136:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2553 00000100:00080000:2.0:1767320328.927523:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320328.927529:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320328.927903:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985397504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320328.933202:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985397632:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320329.058017:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320330.079914:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320331.100538:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320331.433910:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320331.434064:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920611072 00000100:00100000:3.0:1767320331.434073:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920611072 00000100:00100000:3.0:1767320331.434145:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5156 00000100:00100000:3.0:1767320331.434155:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920611072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320331.434282:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920611072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 129us (379us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320331.434290:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5156 00000100:00080000:2.0:1767320331.434348:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320331.436239:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320331.437290:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920610944 00000100:00100000:1.0:1767320331.437301:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920610944 00000100:00100000:1.0:1767320331.437348:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5813 00000100:00100000:1.0:1767320331.437355:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920610944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320331.437379:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320331.437449:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920610944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 93us (1219us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320331.437458:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5813 00000100:00080000:0.0:1767320331.483534:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320331.483545:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320331.483549:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320331.483553:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320331.483557:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320331.483564:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5574 00000100:00080000:0.0:1767320331.483569:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320331.483575:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320331.483614:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ed80 x1853163985397760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320331.483669:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985397760:0@lo:38:kworker.0 00000100:00100000:3.0:1767320331.483737:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320331.484041:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985397760 00000100:00080000:0.0:1767320331.484042:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320331.484046:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320331.484048:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320331.484050:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985397760 00000100:00080000:0.0:1767320331.484051:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5574 00000100:00080000:0.0:1767320331.484063:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320331.484067:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320331.484081:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18f480 x1853163985397888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320331.484156:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5814 00000100:00100000:1.0:1767320331.484184:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985397760:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320331.484234:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320331.488429:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985397888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320331.488806:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320331.488904:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985397888 00000100:00100000:2.0:1767320331.488912:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985397888 00000100:00100000:2.0:1767320331.488953:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1944 00000100:00100000:2.0:1767320331.488979:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985397888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320331.489001:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320331.489044:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985397888:12345-0@lo:38:kworker.0 Request processed in 64us (238us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320331.489052:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1944 00000100:00080000:0.0:1767320331.489569:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320331.489574:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320331.489578:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320331.489583:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5574 00000100:00080000:0.0:1767320331.489587:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320331.489591:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320331.489608:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18f800 x1853163985398144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320331.489625:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320331.489627:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320331.489629:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320331.489632:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5574 00000100:00080000:0.0:1767320331.489634:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320331.489637:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320331.489646:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c700 x1853163985398400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320331.489703:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985398528:0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.489740:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320331.489767:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320331.489772:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985398656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.489792:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320331.489821:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398528 00000100:00100000:0.0:1767320331.489826:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398528 00000100:00100000:0.0:1767320331.489865:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398656 00000100:00100000:0.0:1767320331.489868:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398656 00000100:00100000:0.0:1767320331.489888:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3357 00000100:00100000:0.0:1767320331.489895:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985398528:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.489935:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985398528:12345-0@lo:400:kworker.0 Request processed in 41us (195us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320331.489941:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3357 00000100:00100000:0.0:1767320331.490238:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3358 00000100:00100000:0.0:1767320331.490247:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985398656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.490277:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985398656:12345-0@lo:400:kworker.0 Request processed in 33us (484us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320331.490283:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3358 00000100:00100000:0.0:1767320331.490327:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985398656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.490338:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985398528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.490428:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985398016:0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.490460:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320331.490500:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320331.490505:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985398272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.490523:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320331.490548:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320331.490552:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320331.490562:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398016 00000100:00100000:2.0:1767320331.490566:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398016 00000100:00100000:2.0:1767320331.490594:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398272 00000100:00100000:2.0:1767320331.490597:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398272 00000100:00100000:2.0:1767320331.490618:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5157 00000100:00100000:2.0:1767320331.490625:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985398016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.490658:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985398016:12345-0@lo:400:kworker.0 Request processed in 34us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320331.490664:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5157 00000100:00100000:2.0:1767320331.490676:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5158 00000100:00100000:2.0:1767320331.490681:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985398272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.490705:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985398272:12345-0@lo:400:kworker.0 Request processed in 24us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320331.490710:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5158 00000100:00080000:2.0:1767320331.490723:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320331.490738:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985398272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.490747:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985398016:0@lo:400:kworker.0 00000100:00100000:1.0:1767320331.492660:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985397760:12345-0@lo:38:kworker.0 Request processed in 8485us (8913us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320331.492672:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5814 00000100:00080000:3.0:1767320331.494765:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320331.494778:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320331.494784:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985397760:0@lo:38:kworker.0 00000100:00100000:3.0:1767320331.494827:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985398144:0@lo:38:kworker.0 00000100:00100000:3.0:1767320331.494897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320331.494945:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985398400:0@lo:38:kworker.0 00000100:00100000:3.0:1767320331.495046:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320331.495373:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320331.495379:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320331.495382:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985397888:0@lo:38:kworker.0 00000100:00100000:0.0:1767320331.495448:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398144 00000100:00100000:0.0:1767320331.495455:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398144 00000100:00100000:1.0:1767320331.495475:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398400 00000100:00100000:1.0:1767320331.495480:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398400 00000100:00100000:1.0:1767320331.495527:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5815 00000100:00100000:1.0:1767320331.495534:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985398400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320331.495559:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320331.495616:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985398400:12345-0@lo:38:kworker.0 Request processed in 80us (568us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320331.495623:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5815 00000100:00080000:3.0:1767320331.495660:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320331.495665:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320331.495668:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985398400:0@lo:38:kworker.0 00000100:00100000:0.0:1767320331.495857:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5816 00000100:00100000:0.0:1767320331.495862:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985398144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320331.495874:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320331.495914:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985398144:12345-0@lo:38:kworker.0 Request processed in 50us (1017us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320331.495919:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5816 00000100:00080000:3.0:1767320331.495983:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320331.495988:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320331.495992:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985398144:0@lo:38:kworker.0 00010000:00010000:2.0:1767320332.123465:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320333.149265:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320334.047499:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320334.047523:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985398784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320334.047589:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320334.047805:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398784 00000100:00100000:0.0:1767320334.047814:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398784 00000100:00100000:0.0:1767320334.047880:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2554 00000100:00100000:0.0:1767320334.047890:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985398784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320334.047918:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320334.047961:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985398784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320334.047968:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2554 00000100:00100000:1.0:1767320334.048063:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985398784:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320334.048123:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320334.049319:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320334.049333:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985398912:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320334.049392:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320334.052378:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985398912 00000100:00100000:3.0:1767320334.052392:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985398912 00000100:00100000:3.0:1767320334.052476:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2555 00000100:00100000:3.0:1767320334.052486:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985398912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320334.052518:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320334.052615:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985398912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 130us (3222us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320334.052622:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2555 00000100:00100000:1.0:1767320334.053338:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985398912:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320334.053450:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320334.171414:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320335.195383:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320336.223106:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320336.558005:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320336.558153:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320336.558343:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920611328 00000100:00100000:2.0:1767320336.558355:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920611328 00000100:00100000:2.0:1767320336.558448:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920611200 00000100:00100000:2.0:1767320336.558452:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920611200 00000100:00100000:2.0:1767320336.558477:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5159 00000100:00100000:2.0:1767320336.558491:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920611328:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320336.558531:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5817 00000100:00100000:1.0:1767320336.558542:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920611200:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320336.558570:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320336.558588:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920611328:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (612us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320336.558597:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5159 00000100:00100000:1.0:1767320336.558608:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920611200:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 66us (454us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320336.558618:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5817 00000100:00080000:0.0:1767320336.559365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320336.603712:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320336.603728:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320336.603732:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320336.603737:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320336.603743:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320336.603752:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5579 00000100:00080000:3.0:1767320336.603758:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320336.604041:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320336.604096:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ca80 x1853163985399040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320336.604144:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320336.604148:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320336.604150:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320336.604154:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5579 00000100:00080000:3.0:1767320336.604157:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320336.604208:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320336.604225:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7e300 x1853163985399168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320336.604650:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985399296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320336.604738:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320336.604825:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399296 00000100:00100000:1.0:1767320336.604837:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399296 00000100:00100000:1.0:1767320336.604910:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5160 00000100:00100000:1.0:1767320336.604921:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985399296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.604981:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985399040:0@lo:38:kworker.0 00000100:00100000:0.0:1767320336.605029:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320336.605058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985399168:0@lo:38:kworker.0 00000100:00100000:0.0:1767320336.605087:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320336.605146:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399040 00000100:00100000:0.0:1767320336.605150:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399040 00000100:00100000:0.0:1767320336.605201:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5818 00000100:00100000:0.0:1767320336.605207:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985399040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320336.605229:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320336.605276:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985399040:12345-0@lo:38:kworker.0 Request processed in 56us (233us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320336.605284:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5818 00000100:00100000:0.0:1767320336.605312:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399168 00000100:00100000:0.0:1767320336.605315:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399168 00000100:00100000:0.0:1767320336.605336:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1945 00000100:00100000:0.0:1767320336.605341:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985399168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320336.605350:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320336.605521:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320336.605526:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320336.605529:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320336.605533:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5579 00000100:00080000:3.0:1767320336.605536:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320336.605541:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320336.605562:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ea00 x1853163985399424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320336.605757:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985399168:12345-0@lo:38:kworker.0 Request processed in 415us (669us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320336.605765:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1945 00000100:00100000:0.0:1767320336.605786:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985399552:0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.605813:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320336.605837:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399552 00000100:00100000:0.0:1767320336.605840:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399552 00000100:00100000:0.0:1767320336.605861:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5161 00000100:00100000:0.0:1767320336.605865:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985399552:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.605887:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985399552:12345-0@lo:400:kworker.0 Request processed in 22us (75us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320336.605893:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5161 00000100:00100000:0.0:1767320336.605931:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985399552:0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.605955:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985399424:0@lo:38:kworker.0 00000100:00100000:0.0:1767320336.605973:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320336.605994:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320336.605999:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320336.606002:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985399040:0@lo:38:kworker.0 00000100:00080000:0.0:1767320336.606012:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320336.606015:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320336.606018:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985399168:0@lo:38:kworker.0 00000100:00100000:0.0:1767320336.606039:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399424 00000100:00100000:0.0:1767320336.606042:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399424 00000100:00100000:0.0:1767320336.606060:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5819 00000100:00100000:0.0:1767320336.606065:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985399424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320336.606076:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320336.606103:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985399424:12345-0@lo:38:kworker.0 Request processed in 36us (127us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320336.606123:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5819 00000100:00080000:0.0:1767320336.606144:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320336.606148:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320336.606150:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985399424:0@lo:38:kworker.0 00000100:00080000:0.0:1767320336.606198:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320336.607995:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320336.608006:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320336.608010:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320336.608019:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5579 00000100:00080000:3.0:1767320336.608024:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320336.608031:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320336.608057:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985399296:12345-0@lo:400:kworker.0 Request processed in 3133us (3313us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320336.608073:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5160 00000100:00100000:3.0:1767320336.608076:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ce00 x1853163985399680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320336.608193:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985399680:0@lo:38:kworker.0 00000100:00100000:0.0:1767320336.608262:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320336.608611:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985399296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320336.608704:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985399808:0@lo:400:kworker.0 00000100:00100000:2.0:1767320336.608741:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320336.608832:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985399936:0@lo:400:kworker.0 00000100:00100000:1.0:1767320336.608908:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399680 00000100:00100000:1.0:1767320336.608916:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399680 00000100:00100000:1.0:1767320336.608960:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5820 00000100:00100000:3.0:1767320336.608965:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320336.608972:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985399680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320336.608997:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320336.609122:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399808 00000100:00100000:0.0:1767320336.609127:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399808 00000100:00100000:0.0:1767320336.609196:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985399936 00000100:00100000:0.0:1767320336.609199:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985399936 00000100:00100000:0.0:1767320336.609217:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3359 00000100:00100000:0.0:1767320336.609221:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985399808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.609255:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985399808:12345-0@lo:400:kworker.0 Request processed in 34us (515us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320336.609260:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3359 00000100:00100000:0.0:1767320336.609284:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3360 00000100:00100000:0.0:1767320336.609289:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985399936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.609323:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985399936:12345-0@lo:400:kworker.0 Request processed in 33us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320336.609328:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3360 00000100:00080000:0.0:1767320336.609354:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320336.609358:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320336.609427:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985399680:12345-0@lo:38:kworker.0 Request processed in 455us (1164us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320336.609436:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5820 00000100:00100000:3.0:1767320336.609600:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985399936:0@lo:400:kworker.0 00000100:00100000:3.0:1767320336.609636:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985399808:0@lo:400:kworker.0 00000100:00080000:2.0:1767320336.609707:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320336.609712:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320336.609716:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985399680:0@lo:38:kworker.0 00010000:00010000:0.0:1767320337.243599:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320338.267544:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320339.164109:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985400064:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320339.164224:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320339.164372:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985400192:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320339.166890:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985400064 00000100:00100000:0.0:1767320339.166911:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985400064 00000100:00100000:0.0:1767320339.167020:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2556 00000100:00100000:0.0:1767320339.167031:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985400064:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320339.167082:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:0.0:1767320339.167085:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320339.167199:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320339.167250:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985400192 00000100:00100000:3.0:1767320339.167257:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985400192 00000100:00100000:3.0:1767320339.167302:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2557 00000100:00100000:3.0:1767320339.167312:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985400192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320339.167333:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320339.167405:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985400192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (207us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320339.167414:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2557 00000100:00080000:3.0:1767320339.167436:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320339.168184:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985400064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1150us (3957us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320339.168194:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2556 00000100:00100000:1.0:1767320339.168595:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985400192:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320339.169677:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985400064:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320339.294282:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320340.315843:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320340.653856:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767320340.653889:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920611456 offset 0 length 520: 4 00000100:00100000:3.0:1767320340.653914:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320340.653994:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920611712 00000100:00100000:2.0:1767320340.654011:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920611712 00000100:00100000:2.0:1767320340.654126:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920611584 00000100:00100000:2.0:1767320340.654131:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920611584 00000100:00100000:2.0:1767320340.654159:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5162 00000100:00100000:2.0:1767320340.654223:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920611712:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320340.654328:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920611712:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 111us (482us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320340.654337:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5162 00000100:00100000:2.0:1767320340.654360:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5821 00000100:00100000:2.0:1767320340.654365:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920611584:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320340.654390:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320340.654420:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920611584:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (505us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320340.654426:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5821 00000100:00080000:3.0:1767320340.655347:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320341.340751:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320341.723379:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320316/real 1767320316] req@ffff8ce1a612dc00 x1853163985393792/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320341 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320341.723427:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a612dc00 x1853163985393792/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320341 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320341.723454:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320341.723466:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320341.723471:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985393792:0@lo:250:kworker.0 00000100:00080000:2.0:1767320341.723590:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320341.723596:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320341.723600:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320341.723606:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5564 00000100:00080000:2.0:1767320341.723611:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320341.723618:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320341.723668:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4a80 x1853163985400320/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320341.723698:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320341.723703:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320341.723705:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320341.723710:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5584 00000100:00080000:2.0:1767320341.723713:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320341.723717:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320341.723745:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5c00 x1853163985400448/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320341.723756:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320341.723759:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320341.723761:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320341.723765:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5584 00000100:00080000:2.0:1767320341.723767:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320341.723770:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320341.723784:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7b80 x1853163985400576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320341.723835:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320341.723840:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320341.723842:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320341.723845:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5584 00000100:00080000:2.0:1767320341.723848:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320341.723852:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320341.723870:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3100 x1853163985400832/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320341.723887:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320341.723891:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320341.723893:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320341.723897:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5584 00000100:00080000:2.0:1767320341.723900:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320341.723904:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320341.723916:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595180 x1853163985401088/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320341.723980:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985401216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320341.724074:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320341.724151:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320341.724157:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985401344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320341.724213:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320341.724254:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401216 00000100:00100000:2.0:1767320341.724264:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401216 00000100:00100000:2.0:1767320341.724325:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401344 00000100:00100000:2.0:1767320341.724329:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401344 00000100:00100000:2.0:1767320341.724356:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3361 00000100:00100000:2.0:1767320341.724367:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985401216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320341.724425:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985401216:12345-0@lo:400:kworker.0 Request processed in 60us (351us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320341.724433:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3361 00000100:00100000:2.0:1767320341.724453:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3362 00000100:00100000:2.0:1767320341.724459:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985401344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320341.724487:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985401344:12345-0@lo:400:kworker.0 Request processed in 28us (273us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320341.724493:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3362 00000100:00100000:2.0:1767320341.724541:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985401344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320341.724552:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985401216:0@lo:400:kworker.0 00000100:00080000:3.0:1767320341.725068:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320341.725076:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320341.725102:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985400320:0@lo:250:kworker.0 00000400:00000100:3.0:1767320341.725136:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985400320 offset 0 length 520: 4 00000100:00100000:3.0:1767320341.725192:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985400448:0@lo:38:kworker.0 00000100:00100000:3.0:1767320341.725222:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320341.725246:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985400576:0@lo:38:kworker.0 00000100:00100000:3.0:1767320341.725271:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320341.725287:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985400832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320341.725309:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320341.725329:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985401088:0@lo:38:kworker.0 00000100:00100000:3.0:1767320341.725367:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320341.725406:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985400448 00000100:00100000:3.0:1767320341.725412:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985400448 00000100:00100000:3.0:1767320341.725440:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985400832 00000100:00100000:3.0:1767320341.725443:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985400832 00000100:00100000:3.0:1767320341.725457:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401088 00000100:00100000:3.0:1767320341.725459:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401088 00000100:00100000:3.0:1767320341.725473:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5822 00000100:00100000:3.0:1767320341.725480:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985400448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320341.725504:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320341.725543:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985400448:12345-0@lo:38:kworker.0 Request processed in 63us (320us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320341.725550:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5822 00000100:00100000:3.0:1767320341.725565:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5823 00000100:00100000:3.0:1767320341.725569:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985400832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320341.725579:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320341.725601:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985400832:12345-0@lo:38:kworker.0 Request processed in 30us (291us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320341.725607:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5823 00000100:00100000:3.0:1767320341.725618:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5824 00000100:00100000:3.0:1767320341.725621:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985401088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320341.725633:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320341.725655:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985401088:12345-0@lo:38:kworker.0 Request processed in 32us (288us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320341.725660:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5824 00000100:00080000:3.0:1767320341.725706:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320341.725712:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320341.725716:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985400448:0@lo:38:kworker.0 00000100:00080000:3.0:1767320341.725726:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320341.725729:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320341.725732:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985400832:0@lo:38:kworker.0 00000100:00080000:3.0:1767320341.726131:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320341.726137:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320341.726139:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985401088:0@lo:38:kworker.0 00000100:00100000:1.0:1767320341.726279:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985400960:0@lo:400:kworker.0 00000100:00100000:1.0:1767320341.726320:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320341.726370:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320341.726378:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985400704:0@lo:400:kworker.0 00000100:00100000:1.0:1767320341.726403:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320341.726434:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985400960 00000100:00100000:1.0:1767320341.726438:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985400960 00000100:00100000:1.0:1767320341.726484:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985400704 00000100:00100000:1.0:1767320341.726488:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985400704 00000100:00100000:1.0:1767320341.726505:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5163 00000100:00100000:1.0:1767320341.726511:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985400960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320341.726560:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985400960:12345-0@lo:400:kworker.0 Request processed in 48us (238us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320341.726567:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5163 00000100:00100000:1.0:1767320341.726583:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5164 00000100:00100000:1.0:1767320341.726589:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985400704:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320341.726617:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985400704:12345-0@lo:400:kworker.0 Request processed in 28us (213us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320341.726622:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5164 00000100:00080000:1.0:1767320341.726638:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320341.726658:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985400704:0@lo:400:kworker.0 00000100:00100000:1.0:1767320341.726667:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985400960:0@lo:400:kworker.0 00000100:00100000:0.0:1767320341.727067:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985400576 00000100:00100000:0.0:1767320341.727074:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985400576 00000100:00100000:0.0:1767320341.727120:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1946 00000100:00100000:0.0:1767320341.727126:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985400576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320341.727142:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320341.727213:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985400576:12345-0@lo:38:kworker.0 Request processed in 86us (1940us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320341.727219:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1946 00000100:00080000:3.0:1767320341.727405:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320341.727411:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320341.727415:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985400576:0@lo:38:kworker.0 00010000:00010000:1.0:1767320342.363501:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320343.388502:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320344.283566:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985401600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320344.283663:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320344.283719:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320344.283726:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985401472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320344.283754:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320344.292742:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401600 00000100:00100000:0.0:1767320344.292761:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401600 00000100:00100000:0.0:1767320344.293050:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401472 00000100:00100000:0.0:1767320344.293055:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401472 00000100:00080000:1.0:1767320344.293110:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320344.293117:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320344.293212:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2558 00000100:00100000:3.0:1767320344.293243:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985401600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320344.293300:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320344.293733:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985401600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 506us (10069us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320344.293743:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2558 00000100:00100000:3.0:1767320344.293770:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2559 00000100:00100000:3.0:1767320344.293778:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985401472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320344.293799:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320344.293946:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985401472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 169us (10191us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320344.293954:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2559 00000100:00100000:1.0:1767320344.294088:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985401600:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320344.294133:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985401472:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320344.412433:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320345.436370:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320345.769577:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320345.770790:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320345.771099:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920611968 00000100:00100000:1.0:1767320345.771110:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920611968 00000100:00100000:0.0:1767320345.771192:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920611840 00000100:00100000:0.0:1767320345.771198:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920611840 00000100:00100000:1.0:1767320345.771272:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5165 00000100:00100000:1.0:1767320345.771282:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920611968:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320345.771356:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920611968:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (1798us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320345.771364:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5165 00000100:00100000:1.0:1767320345.771380:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5825 00000100:00100000:1.0:1767320345.771384:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920611840:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320345.771403:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320345.771426:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920611840:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (641us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320345.771432:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5825 00000100:00080000:3.0:1767320345.771719:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320346.459410:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320346.843312:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320346.843322:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320346.843326:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320346.843329:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320346.843334:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320346.843342:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5589 00000100:00080000:2.0:1767320346.843347:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320346.843354:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320346.843397:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594700 x1853163985401728/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320346.843425:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320346.843428:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320346.843430:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320346.843433:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5589 00000100:00080000:2.0:1767320346.843436:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320346.843439:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320346.843458:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2300 x1853163985401856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320346.843508:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320346.843512:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320346.843514:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320346.843517:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5589 00000100:00080000:2.0:1767320346.843519:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320346.843522:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320346.843536:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10a80 x1853163985402112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320346.843550:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320346.843554:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320346.843555:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320346.843558:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5589 00000100:00080000:2.0:1767320346.843561:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320346.843564:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320346.843573:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10000 x1853163985402368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320346.843795:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985402240:0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.843880:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320346.843975:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320346.843982:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985401984:0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844005:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320346.844040:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985402496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844067:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320346.844085:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320346.844091:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985402624:0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844117:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320346.844151:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985402240 00000100:00100000:1.0:1767320346.844158:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985402240 00000100:00100000:1.0:1767320346.844236:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401984 00000100:00100000:1.0:1767320346.844240:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401984 00000100:00100000:1.0:1767320346.844263:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5166 00000100:00100000:1.0:1767320346.844272:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985402240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844324:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985402240:12345-0@lo:400:kworker.0 Request processed in 53us (443us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320346.844331:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5166 00000100:00100000:1.0:1767320346.844348:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5167 00000100:00100000:1.0:1767320346.844354:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985401984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844383:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985401984:12345-0@lo:400:kworker.0 Request processed in 29us (376us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320346.844389:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5167 00000100:00100000:1.0:1767320346.844410:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985402496 00000100:00100000:1.0:1767320346.844413:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985402496 00000100:00100000:1.0:1767320346.844442:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985402624 00000100:00100000:1.0:1767320346.844445:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985402624 00000100:00100000:1.0:1767320346.844465:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3363 00000100:00100000:1.0:1767320346.844472:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985402496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844506:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985402496:12345-0@lo:400:kworker.0 Request processed in 34us (437us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320346.844512:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3363 00000100:00100000:1.0:1767320346.844523:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3364 00000100:00100000:1.0:1767320346.844529:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a05050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985402624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844556:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a05050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985402624:12345-0@lo:400:kworker.0 Request processed in 28us (438us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320346.844562:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3364 00000100:00100000:1.0:1767320346.844789:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985402624:0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844800:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985402496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844828:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985401984:0@lo:400:kworker.0 00000100:00100000:1.0:1767320346.844838:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985402240:0@lo:400:kworker.0 00000100:00100000:3.0:1767320346.845794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985401728:0@lo:38:kworker.0 00000100:00100000:3.0:1767320346.845846:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320346.846014:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320346.846019:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320346.846022:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320346.846136:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985401856:0@lo:38:kworker.0 00000100:00100000:3.0:1767320346.846182:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320346.846330:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985402112:0@lo:38:kworker.0 00000100:00100000:3.0:1767320346.846355:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320346.846379:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985402368:0@lo:38:kworker.0 00000100:00100000:3.0:1767320346.846401:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320346.846432:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401728 00000100:00100000:3.0:1767320346.846437:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401728 00000100:00100000:3.0:1767320346.846464:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985402112 00000100:00100000:3.0:1767320346.846467:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985402112 00000100:00100000:3.0:1767320346.846479:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985402368 00000100:00100000:3.0:1767320346.846482:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985402368 00000100:00100000:3.0:1767320346.846496:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5826 00000100:00100000:3.0:1767320346.846502:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985401728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320346.846521:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320346.846558:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985401728:12345-0@lo:38:kworker.0 Request processed in 54us (711us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320346.846568:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5826 00000100:00100000:3.0:1767320346.846584:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5827 00000100:00100000:3.0:1767320346.846587:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985402112:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320346.846596:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320346.846618:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985402112:12345-0@lo:38:kworker.0 Request processed in 29us (262us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320346.846623:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5827 00000100:00100000:3.0:1767320346.846633:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5828 00000100:00100000:3.0:1767320346.846637:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985402368:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320346.846645:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320346.846665:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985402368:12345-0@lo:38:kworker.0 Request processed in 27us (264us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320346.846671:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5828 00000100:00080000:3.0:1767320346.846710:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320346.846715:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320346.846718:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985401728:0@lo:38:kworker.0 00000100:00080000:3.0:1767320346.846725:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320346.846728:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320346.846731:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985402112:0@lo:38:kworker.0 00000100:00080000:3.0:1767320346.846737:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320346.846740:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320346.846742:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985402368:0@lo:38:kworker.0 00000100:00100000:0.0:1767320346.847340:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985401856 00000100:00100000:0.0:1767320346.847346:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985401856 00000100:00100000:0.0:1767320346.847383:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1947 00000100:00100000:0.0:1767320346.847388:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985401856:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320346.847400:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320346.847442:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985401856:12345-0@lo:38:kworker.0 Request processed in 53us (1277us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320346.847449:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1947 00000100:00080000:0.0:1767320346.847478:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320346.847482:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320346.847485:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985401856:0@lo:38:kworker.0 00010000:00010000:2.0:1767320347.486310:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320348.509586:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320349.404506:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985402752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320349.404591:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320349.404822:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985402752 00000100:00100000:3.0:1767320349.404831:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985402752 00000100:00100000:3.0:1767320349.404902:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2560 00000100:00100000:3.0:1767320349.404911:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985402752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320349.404939:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320349.404978:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985402752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320349.404985:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2560 00000100:00080000:2.0:1767320349.405042:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320349.405086:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985402752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320349.405249:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320349.405258:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985402880:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320349.405287:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320349.405445:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985402880 00000100:00100000:3.0:1767320349.405450:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985402880 00000100:00100000:3.0:1767320349.405495:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2561 00000100:00100000:3.0:1767320349.405502:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985402880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320349.405520:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320349.405552:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985402880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 51us (264us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320349.405558:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2561 00000100:00080000:2.0:1767320349.406074:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320349.406105:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985402880:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320349.531485:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320350.555907:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320350.896149:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320350.896241:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320350.896313:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612096 00000100:00100000:2.0:1767320350.896324:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612096 00000100:00100000:1.0:1767320350.896326:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612224 00000100:00100000:1.0:1767320350.896329:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612224 00000100:00080000:0.0:1767320350.896425:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320350.896426:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5168 00000100:00100000:2.0:1767320350.896438:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920612224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320350.896518:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920612224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320350.896526:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5168 00000100:00100000:2.0:1767320350.896544:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5829 00000100:00100000:2.0:1767320350.896548:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920612096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320350.896566:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320350.896591:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920612096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (452us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320350.896598:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5829 00010000:00010000:1.0:1767320351.579396:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320351.963441:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320351.963451:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320351.963454:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320351.963458:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320351.963463:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320351.963471:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5595 00000100:00080000:3.0:1767320351.963476:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320351.963483:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320351.963533:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0380 x1853163985403008/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320351.963558:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320351.963562:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320351.963564:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320351.963567:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5595 00000100:00080000:3.0:1767320351.963569:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320351.963573:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320351.963586:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1c00 x1853163985403136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320351.963628:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320351.963631:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320351.963633:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320351.963637:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5595 00000100:00080000:3.0:1767320351.963639:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320351.963643:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320351.963655:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1f80 x1853163985403392/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320351.963672:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320351.963675:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320351.963677:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320351.963680:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5595 00000100:00080000:3.0:1767320351.963683:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320351.963686:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320351.963696:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4380 x1853163985403648/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320351.963790:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985403520:0@lo:400:kworker.0 00000100:00100000:2.0:1767320351.963886:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320351.963956:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320351.963962:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985403264:0@lo:400:kworker.0 00000100:00100000:2.0:1767320351.963984:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320351.964012:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985403776:0@lo:400:kworker.0 00000100:00100000:2.0:1767320351.964036:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320351.964066:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320351.964071:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985403904:0@lo:400:kworker.0 00000100:00100000:2.0:1767320351.964090:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320351.964118:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403520 00000100:00100000:2.0:1767320351.964125:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403520 00000100:00100000:2.0:1767320351.964191:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403264 00000100:00100000:2.0:1767320351.964195:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403264 00000100:00100000:2.0:1767320351.964219:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5169 00000100:00100000:2.0:1767320351.964227:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985403520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320351.964275:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985403520:12345-0@lo:400:kworker.0 Request processed in 48us (389us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320351.964282:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5169 00000100:00100000:2.0:1767320351.964300:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5170 00000100:00100000:2.0:1767320351.964306:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985403264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320351.964332:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985403264:12345-0@lo:400:kworker.0 Request processed in 27us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320351.964338:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5170 00000100:00100000:2.0:1767320351.964375:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985403264:0@lo:400:kworker.0 00000100:00100000:2.0:1767320351.964386:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985403520:0@lo:400:kworker.0 00000100:00100000:0.0:1767320351.964616:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985403008:0@lo:38:kworker.0 00000100:00100000:0.0:1767320351.964650:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320351.964684:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985403136:0@lo:38:kworker.0 00000100:00100000:0.0:1767320351.964708:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320351.964730:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985403392:0@lo:38:kworker.0 00000100:00100000:0.0:1767320351.964751:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320351.964770:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985403648:0@lo:38:kworker.0 00000100:00100000:0.0:1767320351.964793:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320351.964838:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403776 00000100:00100000:0.0:1767320351.964843:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403776 00000100:00100000:0.0:1767320351.964872:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403904 00000100:00100000:0.0:1767320351.964874:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403904 00000100:00100000:0.0:1767320351.964892:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3365 00000100:00100000:0.0:1767320351.964898:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985403776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320351.964942:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985403776:12345-0@lo:400:kworker.0 Request processed in 43us (904us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320351.964949:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3365 00000100:00100000:0.0:1767320351.964960:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3366 00000100:00100000:0.0:1767320351.964967:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985403904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320351.964990:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985403904:12345-0@lo:400:kworker.0 Request processed in 25us (899us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320351.964995:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3366 00000100:00080000:0.0:1767320351.965008:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320351.965012:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320351.965015:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320351.965024:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403008 00000100:00100000:0.0:1767320351.965027:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403008 00000100:00100000:3.0:1767320351.965195:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403392 00000100:00100000:3.0:1767320351.965199:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403392 00000100:00100000:3.0:1767320351.965213:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403648 00000100:00100000:3.0:1767320351.965215:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403648 00000100:00100000:3.0:1767320351.965224:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5830 00000100:00100000:3.0:1767320351.965229:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985403008:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320351.965246:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320351.965281:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985403008:12345-0@lo:38:kworker.0 Request processed in 52us (631us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320351.965287:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5830 00000100:00100000:3.0:1767320351.965298:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5831 00000100:00100000:3.0:1767320351.965301:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985403392:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320351.965310:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320351.965332:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985403392:12345-0@lo:38:kworker.0 Request processed in 29us (580us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320351.965337:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5831 00000100:00100000:3.0:1767320351.965349:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5832 00000100:00100000:3.0:1767320351.965352:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985403648:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320351.965360:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320351.965380:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985403648:12345-0@lo:38:kworker.0 Request processed in 26us (587us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320351.965384:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5832 00000100:00100000:3.0:1767320351.965438:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985403904:0@lo:400:kworker.0 00000100:00100000:3.0:1767320351.965449:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985403776:0@lo:400:kworker.0 00000100:00080000:3.0:1767320351.965485:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320351.965490:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320351.965493:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985403008:0@lo:38:kworker.0 00000100:00080000:3.0:1767320351.965501:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320351.965505:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320351.965507:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985403392:0@lo:38:kworker.0 00000100:00080000:3.0:1767320351.965514:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320351.965517:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320351.965519:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985403648:0@lo:38:kworker.0 00000100:00100000:2.0:1767320351.969270:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985403136 00000100:00100000:2.0:1767320351.969285:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985403136 00000100:00100000:2.0:1767320351.969340:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1948 00000100:00100000:2.0:1767320351.969348:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985403136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320351.969371:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320351.969455:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985403136:12345-0@lo:38:kworker.0 Request processed in 106us (4745us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320351.969466:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1948 00000100:00080000:3.0:1767320351.973071:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320351.973085:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320351.973090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985403136:0@lo:38:kworker.0 00010000:00010000:1.0:1767320352.603441:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320353.628533:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320354.523813:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985404032:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320354.523883:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320354.523997:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404032 00000100:00100000:3.0:1767320354.524005:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404032 00000100:00100000:3.0:1767320354.524064:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2562 00000100:00100000:3.0:1767320354.524072:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985404032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320354.524098:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320354.524141:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985404032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (258us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320354.524148:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2562 00000100:00080000:1.0:1767320354.524230:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320354.524265:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985404032:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320354.524397:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320354.524406:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985404160:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320354.524432:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320354.524507:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404160 00000100:00100000:3.0:1767320354.524511:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404160 00000100:00100000:3.0:1767320354.524551:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2563 00000100:00100000:3.0:1767320354.524559:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985404160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320354.524578:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320354.524609:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985404160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 52us (177us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320354.524615:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2563 00000100:00080000:1.0:1767320354.524671:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320354.524691:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985404160:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320354.651578:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320355.677213:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320356.011788:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320356.011835:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320356.011934:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612480 00000100:00100000:2.0:1767320356.011943:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612480 00000100:00100000:2.0:1767320356.012013:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612352 00000100:00100000:2.0:1767320356.012016:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612352 00000100:00100000:2.0:1767320356.012034:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5171 00000100:00100000:2.0:1767320356.012045:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920612480:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320356.012112:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920612480:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 70us (333us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320356.012119:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5171 00000100:00100000:2.0:1767320356.012138:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5833 00000100:00100000:2.0:1767320356.012143:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920612352:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320356.012198:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320356.012223:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920612352:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 78us (388us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320356.012229:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5833 00000100:00080000:0.0:1767320356.012860:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320356.701551:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320357.083557:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320357.083567:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320357.083571:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320357.083575:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320357.083578:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320357.083586:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5600 00000100:00080000:3.0:1767320357.083590:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320357.083597:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320357.083635:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357aa00 x1853163985404288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320357.083665:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320357.083676:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320357.083678:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320357.083682:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5600 00000100:00080000:3.0:1767320357.083684:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320357.083687:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320357.083702:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579180 x1853163985404416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320357.083752:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320357.083756:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320357.083758:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320357.083763:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5600 00000100:00080000:3.0:1767320357.083765:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320357.083769:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320357.083784:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985404288:0@lo:38:kworker.0 00000100:00100000:3.0:1767320357.083784:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d500 x1853163985404672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320357.083810:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320357.083813:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320357.083815:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320357.083818:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5600 00000100:00080000:3.0:1767320357.083820:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320357.083823:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320357.083832:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985404928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320357.083860:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320357.083881:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985405184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320357.083920:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320357.083925:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985404416:0@lo:38:kworker.0 00000100:00100000:3.0:1767320357.083946:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320357.083953:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985405056:0@lo:400:kworker.0 00000100:00100000:0.0:1767320357.083960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320357.083989:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320357.084004:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985404672:0@lo:38:kworker.0 00000100:00100000:0.0:1767320357.084032:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320357.084037:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404416 00000100:00100000:3.0:1767320357.084044:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404416 00000100:00100000:0.0:1767320357.084054:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985404928:0@lo:38:kworker.0 00000100:00100000:3.0:1767320357.084077:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1949 00000100:00100000:0.0:1767320357.084083:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320357.084088:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985404416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320357.084108:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320357.084140:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404288 00000100:00100000:0.0:1767320357.084145:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404288 00000100:00100000:3.0:1767320357.084154:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985404416:12345-0@lo:38:kworker.0 Request processed in 67us (196us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320357.084187:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1949 00000100:00100000:0.0:1767320357.084189:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404672 00000100:00100000:0.0:1767320357.084193:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404672 00000100:00100000:0.0:1767320357.084212:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404928 00000100:00100000:0.0:1767320357.084214:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404928 00000100:00100000:3.0:1767320357.084221:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5834 00000100:00100000:3.0:1767320357.084289:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985404288:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320357.084301:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320357.084329:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985404288:12345-0@lo:38:kworker.0 Request processed in 102us (471us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320357.084344:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985404800:0@lo:400:kworker.0 00000100:00100000:1.0:1767320357.084376:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320357.084403:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320357.084408:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985404544:0@lo:400:kworker.0 00000100:00100000:1.0:1767320357.084428:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320357.084447:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405184 00000100:00100000:1.0:1767320357.084450:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405184 00000100:00100000:1.0:1767320357.084488:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405056 00000100:00100000:1.0:1767320357.084491:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405056 00000100:00100000:1.0:1767320357.084511:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3367 00000100:00100000:1.0:1767320357.084517:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985405184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320357.084558:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985405184:12345-0@lo:400:kworker.0 Request processed in 41us (638us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320357.084564:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3367 00000100:00100000:1.0:1767320357.084577:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3368 00000100:00100000:1.0:1767320357.084583:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985405056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320357.084609:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985405056:12345-0@lo:400:kworker.0 Request processed in 26us (622us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320357.084616:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3368 00000100:00080000:1.0:1767320357.084634:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320357.084638:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320357.084653:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404800 00000100:00100000:1.0:1767320357.084657:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404800 00000100:00100000:0.0:1767320357.086666:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985404544 00000100:00100000:0.0:1767320357.086681:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985404544 00000100:00100000:0.0:1767320357.086724:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5172 00000100:00100000:0.0:1767320357.086745:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985404544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320357.086776:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5835 00000100:00100000:2.0:1767320357.086784:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985404672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320357.086803:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320357.086830:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985404544:12345-0@lo:400:kworker.0 Request processed in 97us (2399us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320357.086840:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5172 00000100:00100000:2.0:1767320357.086846:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985404672:12345-0@lo:38:kworker.0 Request processed in 61us (2813us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320357.086853:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5835 00000100:00100000:0.0:1767320357.086862:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5836 00000100:00100000:0.0:1767320357.086868:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985404928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320357.086881:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320357.086915:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985404928:12345-0@lo:38:kworker.0 Request processed in 45us (2833us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320357.086923:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5836 00000100:00080000:2.0:1767320357.086927:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320357.086934:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320357.086939:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985404672:0@lo:38:kworker.0 00000100:00080000:2.0:1767320357.086949:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320357.086952:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320357.086955:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985404928:0@lo:38:kworker.0 00000100:00080000:2.0:1767320357.086967:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320357.086971:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320357.086974:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985404288:0@lo:38:kworker.0 00000100:00080000:2.0:1767320357.086990:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320357.086993:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320357.086997:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985405056:0@lo:400:kworker.0 00000100:00100000:2.0:1767320357.086999:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985404416:0@lo:38:kworker.0 00000100:00100000:0.0:1767320357.087022:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985405184:0@lo:400:kworker.0 00000100:00080000:2.0:1767320357.087043:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320357.087066:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985404544:0@lo:400:kworker.0 00000100:00100000:3.0:1767320357.087325:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5834 00000100:00100000:1.0:1767320357.094378:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5173 00000100:00100000:1.0:1767320357.094417:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985404800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320357.094516:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985404800:12345-0@lo:400:kworker.0 Request processed in 114us (10139us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320357.094524:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5173 00000100:00100000:2.0:1767320357.094583:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985404800:0@lo:400:kworker.0 00010000:00010000:1.0:1767320357.723525:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320358.747748:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320359.643656:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320359.643674:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985405312:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320359.643767:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320359.644019:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405312 00000100:00100000:3.0:1767320359.644027:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405312 00000100:00100000:3.0:1767320359.644088:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2564 00000100:00100000:3.0:1767320359.644096:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985405312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320359.644133:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320359.644193:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985405312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (426us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320359.644211:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2564 00000100:00080000:1.0:1767320359.644387:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320359.644423:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985405312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320359.645212:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320359.645225:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985405440:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320359.645268:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320359.645443:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405440 00000100:00100000:3.0:1767320359.645449:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405440 00000100:00100000:3.0:1767320359.645503:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2565 00000100:00100000:3.0:1767320359.645511:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985405440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320359.645543:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320359.645597:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985405440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (328us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320359.645604:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2565 00000100:00100000:2.0:1767320359.645805:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985405440:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320359.645905:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320359.771981:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320360.795414:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320361.134698:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320361.134810:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320361.134989:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612736 00000100:00100000:3.0:1767320361.134999:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612736 00000100:00100000:3.0:1767320361.135072:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612608 00000100:00100000:3.0:1767320361.135076:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612608 00000100:00100000:3.0:1767320361.135096:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5174 00000100:00100000:3.0:1767320361.135107:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920612736:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320361.135360:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5837 00000100:00100000:1.0:1767320361.135369:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce187fe8050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920612608:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320361.135389:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320361.135438:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce187fe8050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920612608:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 68us (627us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320361.135445:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5837 00000100:00100000:3.0:1767320361.135480:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920612736:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 374us (823us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320361.135495:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5174 00000100:00080000:2.0:1767320361.136711:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320361.819389:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320362.204325:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320362.204333:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320362.204336:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320362.204340:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320362.204344:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320362.204351:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5605 00000100:00080000:3.0:1767320362.204356:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320362.204362:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320362.204399:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d880 x1853163985405568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320362.204426:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320362.204429:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320362.204431:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320362.204435:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5605 00000100:00080000:3.0:1767320362.204437:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320362.204440:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320362.204452:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f100 x1853163985405696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320362.204538:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985405568:0@lo:38:kworker.0 00000100:00080000:3.0:1767320362.204563:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320362.204566:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320362.204568:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320362.204571:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5605 00000100:00080000:3.0:1767320362.204574:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320362.204577:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320362.204587:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d180 x1853163985405952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320362.204599:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320362.204600:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320362.204602:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320362.204604:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320362.204607:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5605 00000100:00080000:3.0:1767320362.204609:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320362.204612:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320362.204620:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c700 x1853163985406208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320362.204682:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985405696:0@lo:38:kworker.0 00000100:00100000:2.0:1767320362.204707:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320362.204723:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985406464:0@lo:400:kworker.0 00000100:00100000:2.0:1767320362.204728:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985405952:0@lo:38:kworker.0 00000100:00100000:2.0:1767320362.204746:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320362.204749:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320362.204760:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985406208:0@lo:38:kworker.0 00000100:00100000:1.0:1767320362.204767:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320362.204772:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985406336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320362.204776:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320362.204791:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320362.204808:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405696 00000100:00100000:1.0:1767320362.204812:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405568 00000100:00100000:1.0:1767320362.204814:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405568 00000100:00100000:2.0:1767320362.204814:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405696 00000100:00100000:1.0:1767320362.204842:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405952 00000100:00100000:1.0:1767320362.204844:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405952 00000100:00100000:2.0:1767320362.204846:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1950 00000100:00100000:2.0:1767320362.204853:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985405696:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320362.204855:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985406208 00000100:00100000:1.0:1767320362.204857:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985406208 00000100:00100000:1.0:1767320362.204870:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5838 00010000:00020000:2.0:1767320362.204872:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320362.204874:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985405568:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320362.204882:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320362.204965:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5839 00000100:00100000:0.0:1767320362.204971:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985405952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320362.204980:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320362.205016:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985405696:12345-0@lo:38:kworker.0 Request processed in 53us (198us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320362.205024:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1950 00000100:00100000:2.0:1767320362.205047:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5840 00000100:00100000:2.0:1767320362.205051:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985406208:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320362.205060:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320362.205742:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985406080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320362.205773:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320362.205792:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320362.205798:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985405824:0@lo:400:kworker.0 00000100:00100000:3.0:1767320362.205816:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320362.205834:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985406080 00000100:00100000:3.0:1767320362.205839:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985406080 00000100:00100000:3.0:1767320362.205873:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985405824 00000100:00100000:3.0:1767320362.205876:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985405824 00000100:00100000:3.0:1767320362.205890:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5175 00000100:00100000:3.0:1767320362.205896:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985406080:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320362.206678:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985406080:12345-0@lo:400:kworker.0 Request processed in 38us (159us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320362.206689:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5175 00000100:00100000:3.0:1767320362.206714:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5176 00000100:00100000:3.0:1767320362.206723:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985405824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320362.206765:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985405824:12345-0@lo:400:kworker.0 Request processed in 45us (948us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320362.206771:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5176 00000100:00080000:3.0:1767320362.206789:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320362.206821:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985405824:0@lo:400:kworker.0 00000100:00100000:3.0:1767320362.206831:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985406080:0@lo:400:kworker.0 00000100:00100000:1.0:1767320362.208264:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985405568:12345-0@lo:38:kworker.0 Request processed in 3381us (3657us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320362.208276:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5838 00000100:00100000:1.0:1767320362.208365:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985406464 00000100:00100000:1.0:1767320362.208372:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985406464 00000100:00100000:1.0:1767320362.208418:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985406336 00000100:00100000:1.0:1767320362.208421:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985406336 00000100:00100000:1.0:1767320362.208438:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3369 00000100:00100000:1.0:1767320362.208444:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985406464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320362.208500:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985406464:12345-0@lo:400:kworker.0 Request processed in 56us (3749us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320362.208505:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3369 00000100:00100000:1.0:1767320362.208516:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3370 00000100:00100000:1.0:1767320362.208521:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985406336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320362.208544:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985406336:12345-0@lo:400:kworker.0 Request processed in 23us (3752us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320362.208551:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3370 00000100:00100000:1.0:1767320362.208606:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985406336:0@lo:400:kworker.0 00000100:00100000:1.0:1767320362.208617:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985406464:0@lo:400:kworker.0 00000100:00100000:0.0:1767320362.209298:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985405952:12345-0@lo:38:kworker.0 Request processed in 4323us (4547us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320362.209308:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5839 00000100:00080000:0.0:1767320362.209339:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320362.209343:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320362.212614:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985406208:12345-0@lo:38:kworker.0 Request processed in 7550us (7825us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320362.212628:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5840 00000100:00080000:3.0:1767320362.221191:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320362.221205:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320362.221210:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985405696:0@lo:38:kworker.0 00000100:00080000:3.0:1767320362.221234:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320362.221238:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320362.221241:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985405952:0@lo:38:kworker.0 00000100:00080000:3.0:1767320362.221251:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320362.221254:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320362.221257:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985406208:0@lo:38:kworker.0 00000100:00080000:3.0:1767320362.221267:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320362.221272:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320362.221274:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985405568:0@lo:38:kworker.0 00010000:00010000:2.0:1767320362.846594:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320363.868290:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320364.763640:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320364.763656:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985406592:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320364.763717:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320364.763828:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985406592 00000100:00100000:0.0:1767320364.763835:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985406592 00000100:00100000:0.0:1767320364.763890:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2566 00000100:00100000:0.0:1767320364.763898:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985406592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320364.763923:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320364.763962:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985406592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 64us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320364.763970:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2566 00000100:00100000:0.0:1767320364.764006:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985406592:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320364.764178:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320364.764326:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320364.764334:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985406720:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320364.764360:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320364.764421:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985406720 00000100:00100000:2.0:1767320364.764424:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985406720 00000100:00100000:2.0:1767320364.764459:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2567 00000100:00100000:2.0:1767320364.764466:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985406720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320364.764479:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320364.764504:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985406720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 40us (144us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320364.764510:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2567 00000100:00080000:1.0:1767320364.764554:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320364.764573:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985406720:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320364.891925:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320365.939351:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320366.251838:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320366.251921:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320366.252137:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612992 00000100:00100000:2.0:1767320366.252149:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612992 00000100:00100000:1.0:1767320366.252277:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920612864 00000100:00100000:1.0:1767320366.252282:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920612864 00000100:00100000:1.0:1767320366.252331:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5841 00000100:00100000:1.0:1767320366.252338:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920612864:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320366.252361:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320366.252423:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920612864:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 84us (501us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320366.252430:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5841 00000100:00100000:2.0:1767320366.253185:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5177 00000100:00100000:2.0:1767320366.253197:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920612992:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320366.253251:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920612992:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 58us (1423us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320366.253258:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5177 00000400:00000100:2.0:1767320366.253560:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920613120 offset 0 length 520: 4 00000100:00080000:0.0:1767320366.254553:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320366.299441:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320341/real 1767320341] req@ffff8ce1a61c4a80 x1853163985400320/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320366 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320366.299474:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61c4a80 x1853163985400320/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320366 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320366.299492:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320366.299518:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320366.299521:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985400320:0@lo:250:kworker.0 00000100:00080000:2.0:1767320366.299618:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320366.299624:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320366.299628:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320366.299633:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5589 00000100:00080000:2.0:1767320366.299636:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320366.299643:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320366.299677:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985406848/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320366.299708:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320366.299712:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320366.299714:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320366.299718:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5610 00000100:00080000:2.0:1767320366.299722:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320366.299725:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320366.299741:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10e00 x1853163985406976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320366.299751:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320366.299754:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320366.299755:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320366.299758:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5610 00000100:00080000:2.0:1767320366.299761:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320366.299764:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320366.299775:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12680 x1853163985407104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320366.299819:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320366.299823:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320366.299824:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320366.299827:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5610 00000100:00080000:2.0:1767320366.299829:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320366.299833:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320366.299843:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10700 x1853163985407360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320366.299858:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320366.299861:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320366.299862:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320366.299866:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5610 00000100:00080000:2.0:1767320366.299868:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320366.299871:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320366.299880:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357bb80 x1853163985407616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320366.300010:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985407488:0@lo:400:kworker.0 00000100:00100000:1.0:1767320366.300073:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320366.300131:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320366.300137:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985407232:0@lo:400:kworker.0 00000100:00100000:1.0:1767320366.300160:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320366.300224:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985407488 00000100:00100000:1.0:1767320366.300232:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985407488 00000100:00100000:1.0:1767320366.300279:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985407232 00000100:00100000:1.0:1767320366.300282:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985407232 00000100:00100000:1.0:1767320366.300305:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5178 00000100:00100000:1.0:1767320366.300313:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985407488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320366.300358:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985407488:12345-0@lo:400:kworker.0 Request processed in 45us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320366.300365:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5178 00000100:00100000:1.0:1767320366.300380:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5179 00000100:00100000:1.0:1767320366.300387:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985407232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320366.300409:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985407232:12345-0@lo:400:kworker.0 Request processed in 23us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320366.300415:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5179 00000100:00100000:1.0:1767320366.300448:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985407232:0@lo:400:kworker.0 00000100:00100000:1.0:1767320366.300457:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985407488:0@lo:400:kworker.0 00000100:00100000:2.0:1767320366.300861:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985407744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320366.300864:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985407872:0@lo:400:kworker.0 00000100:00100000:2.0:1767320366.301058:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320366.301100:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985407744 00000100:00100000:2.0:1767320366.301105:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985407744 00000100:00100000:2.0:1767320366.301140:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3371 00000100:00100000:2.0:1767320366.301149:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985407744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320366.301226:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985406848:0@lo:250:kworker.0 00000400:00000100:3.0:1767320366.301252:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985406848 offset 0 length 520: 4 00000100:00100000:3.0:1767320366.301271:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985406976:0@lo:38:kworker.0 00000100:00100000:3.0:1767320366.301294:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320366.301313:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985407104:0@lo:38:kworker.0 00000100:00100000:3.0:1767320366.301336:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320366.301352:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985407360:0@lo:38:kworker.0 00000100:00100000:3.0:1767320366.301372:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320366.301388:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985407616:0@lo:38:kworker.0 00000100:00100000:3.0:1767320366.301406:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320366.301436:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985407104 00000100:00100000:3.0:1767320366.301440:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985407104 00000100:00100000:3.0:1767320366.301461:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1951 00000100:00100000:3.0:1767320366.301467:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985407104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320366.301486:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320366.301518:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985407104:12345-0@lo:38:kworker.0 Request processed in 50us (181us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320366.301524:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1951 00000100:00100000:3.0:1767320366.301545:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985406976 00000100:00100000:3.0:1767320366.301549:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985406976 00000100:00100000:1.0:1767320366.301580:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320366.301617:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985407872 00000100:00100000:1.0:1767320366.301621:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985407872 00000100:00100000:1.0:1767320366.301650:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3372 00000100:00100000:1.0:1767320366.301657:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985407872:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320366.301691:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985407744:12345-0@lo:400:kworker.0 Request processed in 542us (631us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320366.301700:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3371 00000100:00100000:1.0:1767320366.301706:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985407872:12345-0@lo:400:kworker.0 Request processed in 47us (139us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320366.301714:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3372 00000100:00080000:0.0:1767320366.301815:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320366.301820:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320366.301821:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320366.301866:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985407872:0@lo:400:kworker.0 00000100:00100000:2.0:1767320366.301926:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985407744:0@lo:400:kworker.0 00000100:00080000:2.0:1767320366.301977:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320366.301982:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320366.301985:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985407104:0@lo:38:kworker.0 00000100:00100000:3.0:1767320366.302065:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985407360 00000100:00100000:1.0:1767320366.302069:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985407616 00000100:00100000:3.0:1767320366.302071:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985407360 00000100:00100000:1.0:1767320366.302075:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985407616 00000100:00100000:3.0:1767320366.302097:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5842 00000100:00100000:3.0:1767320366.302103:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985406976:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320366.302105:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5843 00000100:00100000:1.0:1767320366.302113:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985407360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320366.302121:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320366.302123:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320366.302181:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985407360:12345-0@lo:38:kworker.0 Request processed in 67us (807us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320366.302187:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5843 00000100:00100000:1.0:1767320366.302200:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5844 00000100:00100000:1.0:1767320366.302204:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985407616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320366.302213:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320366.302239:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985407616:12345-0@lo:38:kworker.0 Request processed in 32us (831us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320366.302245:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5844 00000100:00100000:3.0:1767320366.302254:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985406976:12345-0@lo:38:kworker.0 Request processed in 148us (957us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320366.302263:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5842 00000100:00080000:1.0:1767320366.302283:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320366.302288:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320366.302292:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985406976:0@lo:38:kworker.0 00000100:00080000:1.0:1767320366.302300:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320366.302303:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320366.302304:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985407360:0@lo:38:kworker.0 00000100:00080000:1.0:1767320366.302309:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320366.302311:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320366.302312:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985407616:0@lo:38:kworker.0 00010000:00010000:1.0:1767320366.943410:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320367.963414:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320368.987725:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320369.883680:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320369.885512:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985408128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320369.885581:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320369.885625:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985408000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320369.885696:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320369.885913:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408128 00000100:00100000:3.0:1767320369.885921:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408128 00000100:00100000:3.0:1767320369.885998:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408000 00000100:00100000:3.0:1767320369.886002:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408000 00000100:00100000:3.0:1767320369.886022:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2568 00000100:00100000:3.0:1767320369.886031:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985408128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320369.886061:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320369.886102:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985408128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (521us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320369.886109:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2568 00000100:00100000:3.0:1767320369.886124:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2569 00000100:00100000:3.0:1767320369.886130:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985408000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320369.886393:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320369.886425:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985408000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 295us (728us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320369.886432:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2569 00000100:00080000:1.0:1767320369.886579:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320369.886583:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320369.886620:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985408128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320369.886633:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985408000:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320370.011403:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320371.039627:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320371.370499:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320371.370549:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320371.370622:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920613376 00000100:00100000:0.0:1767320371.370632:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920613376 00000100:00100000:0.0:1767320371.370707:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920613248 00000100:00100000:0.0:1767320371.370711:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920613248 00000100:00100000:0.0:1767320371.370730:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5180 00000100:00100000:0.0:1767320371.370741:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920613376:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320371.370817:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920613376:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (329us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320371.370824:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5180 00000100:00100000:0.0:1767320371.370844:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5845 00000100:00100000:0.0:1767320371.370848:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920613248:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320371.370868:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320371.370892:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920613248:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (343us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320371.370910:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5845 00000100:00080000:2.0:1767320371.371669:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320371.419541:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.419552:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320371.419556:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.419561:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.419565:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.419574:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.419580:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767320371.419587:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320371.419639:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579500 x1853163985408256/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320371.419677:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.419681:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.419683:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.419686:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.419689:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767320371.419692:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320371.419706:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a300 x1853163985408384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320371.419787:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767320371.419808:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.419811:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.419813:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.419825:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.419829:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767320371.419832:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320371.419845:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985408640/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320371.419929:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320371.419941:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985408768:0@lo:400:kworker.0 00000100:00100000:0.0:1767320371.420007:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.421015:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408768 00000100:00100000:1.0:1767320371.421026:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408768 00000100:00100000:1.0:1767320371.421100:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5181 00000100:00100000:1.0:1767320371.421110:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985408768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320371.421201:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985408768:12345-0@lo:400:kworker.0 Request processed in 92us (1193us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320371.421209:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5181 00000100:00100000:1.0:1767320371.421245:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985408256:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.421276:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.421303:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985408384:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.421328:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.421351:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985408640:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.421370:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.421399:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408384 00000100:00100000:1.0:1767320371.421403:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408384 00000100:00100000:1.0:1767320371.421421:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1952 00000100:00100000:1.0:1767320371.421425:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985408384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320371.421445:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320371.421505:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985408384:12345-0@lo:38:kworker.0 Request processed in 78us (175us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320371.421511:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1952 00000100:00100000:1.0:1767320371.421530:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408256 00000100:00100000:1.0:1767320371.421533:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408256 00000100:00100000:1.0:1767320371.421548:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408640 00000100:00100000:1.0:1767320371.421550:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408640 00000100:00100000:1.0:1767320371.421564:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5846 00000100:00100000:1.0:1767320371.421568:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985408256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320371.421577:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320371.421609:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985408256:12345-0@lo:38:kworker.0 Request processed in 31us (324us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320371.421615:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5846 00000100:00100000:1.0:1767320371.421625:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5847 00000100:00100000:1.0:1767320371.421629:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985408640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320371.421638:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320371.421668:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985408640:12345-0@lo:38:kworker.0 Request processed in 38us (297us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320371.421674:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5847 00000100:00100000:1.0:1767320371.421712:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985408768:0@lo:400:kworker.0 00000100:00080000:1.0:1767320371.421740:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320371.421745:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320371.421748:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985408256:0@lo:38:kworker.0 00000100:00080000:1.0:1767320371.421907:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320371.421913:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320371.421916:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985408384:0@lo:38:kworker.0 00000100:00080000:1.0:1767320371.421927:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320371.421930:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320371.421932:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985408640:0@lo:38:kworker.0 00000100:00080000:0.0:1767320371.423121:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320371.432516:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985408512:0@lo:400:kworker.0 00000100:00100000:3.0:1767320371.432593:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320371.433057:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.433063:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.433066:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.433071:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.433076:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767320371.433090:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320371.433119:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578700 x1853163985408896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320371.433204:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985409152:0@lo:400:kworker.0 00000100:00100000:2.0:1767320371.433236:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320371.433259:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320371.433265:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985409024:0@lo:400:kworker.0 00000100:00100000:2.0:1767320371.433289:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320371.433319:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409152 00000100:00100000:2.0:1767320371.433325:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409152 00000100:00100000:2.0:1767320371.433372:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409024 00000100:00100000:2.0:1767320371.433375:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409024 00000100:00080000:0.0:1767320371.433665:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320371.433670:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320371.433891:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408512 00000100:00100000:1.0:1767320371.433898:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408512 00000100:00100000:1.0:1767320371.433938:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5182 00000100:00100000:1.0:1767320371.433947:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985408512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320371.434004:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985408512:12345-0@lo:400:kworker.0 Request processed in 58us (1411us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320371.434010:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5182 00000100:00080000:1.0:1767320371.434026:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320371.434046:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985408896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.434070:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.434107:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985408896 00000100:00100000:1.0:1767320371.434111:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985408896 00000100:00100000:1.0:1767320371.434129:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5848 00000100:00100000:1.0:1767320371.434134:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985408896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320371.434161:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320371.434209:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985408896:12345-0@lo:38:kworker.0 Request processed in 73us (138us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320371.434216:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5848 00000100:00080000:1.0:1767320371.434244:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320371.434249:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320371.434252:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985408896:0@lo:38:kworker.0 00000100:00100000:2.0:1767320371.435037:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3373 00000100:00100000:2.0:1767320371.435053:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985409152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320371.435112:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985409152:12345-0@lo:400:kworker.0 Request processed in 65us (1874us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320371.435634:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3374 00000100:00100000:0.0:1767320371.435655:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985409024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320371.435697:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985409024:12345-0@lo:400:kworker.0 Request processed in 55us (2407us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320371.435703:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3374 00000100:00100000:0.0:1767320371.435751:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985409024:0@lo:400:kworker.0 00000100:00100000:0.0:1767320371.435776:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985409152:0@lo:400:kworker.0 00000100:00100000:3.0:1767320371.436426:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985408512:0@lo:400:kworker.0 00000100:00100000:2.0:1767320371.436580:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3373 00010000:00010000:2.0:1767320372.059756:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320373.084890:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320374.107750:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320375.004077:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985409280:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320375.004149:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320375.004596:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409280 00000100:00100000:2.0:1767320375.004603:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409280 00000100:00100000:2.0:1767320375.004664:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2570 00000100:00100000:2.0:1767320375.004672:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985409280:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320375.004700:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320375.004744:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985409280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (595us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320375.004761:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2570 00000100:00080000:1.0:1767320375.004895:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320375.004940:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985409280:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320375.005659:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320375.005672:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985409408:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320375.005714:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320375.005906:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409408 00000100:00100000:0.0:1767320375.005913:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409408 00000100:00100000:0.0:1767320375.005965:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2571 00000100:00100000:0.0:1767320375.005974:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985409408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320375.005994:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320375.006029:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985409408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (314us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320375.006035:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2571 00000100:00080000:1.0:1767320375.006244:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320375.006279:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985409408:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320375.133934:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320376.155596:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320376.490726:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320376.490831:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920613504 00000100:00100000:1.0:1767320376.490841:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920613504 00000100:00100000:1.0:1767320376.490911:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5849 00000100:00100000:1.0:1767320376.490919:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920613504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320376.490944:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320376.491016:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920613504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 95us (300us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320376.491025:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5849 00000100:00100000:0.0:1767320376.499151:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320376.499275:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920613632 00000100:00100000:1.0:1767320376.499284:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920613632 00000100:00100000:1.0:1767320376.499350:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5183 00000100:00100000:1.0:1767320376.499360:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920613632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320376.499447:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920613632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (313us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320376.499456:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5183 00000100:00080000:0.0:1767320376.499607:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320376.539691:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.539703:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320376.539706:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.539709:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.539714:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.539723:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5619 00000100:00080000:2.0:1767320376.539729:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320376.539737:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320376.539777:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b480 x1853163985409536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320376.539794:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.539798:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.539799:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.539803:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5619 00000100:00080000:2.0:1767320376.539806:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320376.539809:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320376.539821:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a680 x1853163985409664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320376.539865:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.539869:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.539871:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.539875:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5619 00000100:00080000:2.0:1767320376.539877:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320376.539880:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320376.539891:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985409920/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320376.539905:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320376.540040:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985409536:0@lo:38:kworker.0 00000100:00100000:1.0:1767320376.540122:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320376.540202:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985409664:0@lo:38:kworker.0 00000100:00100000:1.0:1767320376.540228:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320376.540252:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985409920:0@lo:38:kworker.0 00000100:00100000:1.0:1767320376.540271:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320376.540306:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409536 00000100:00100000:1.0:1767320376.540313:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409536 00000100:00100000:1.0:1767320376.540341:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409920 00000100:00100000:1.0:1767320376.540344:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409920 00000100:00100000:1.0:1767320376.540360:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5850 00000100:00100000:1.0:1767320376.540367:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985409536:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320376.540396:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320376.540434:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985409536:12345-0@lo:38:kworker.0 Request processed in 66us (312us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320376.540446:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5850 00000100:00100000:1.0:1767320376.540462:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5851 00000100:00100000:1.0:1767320376.540467:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985409920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320376.540476:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320376.540786:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985409920:12345-0@lo:38:kworker.0 Request processed in 317us (513us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320376.540792:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5851 00000100:00100000:1.0:1767320376.540813:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409664 00000100:00100000:1.0:1767320376.540817:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409664 00000100:00100000:1.0:1767320376.540837:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1953 00000100:00100000:1.0:1767320376.540841:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985409664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320376.540851:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320376.540875:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985409664:12345-0@lo:38:kworker.0 Request processed in 32us (646us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320376.540880:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1953 00000100:00080000:1.0:1767320376.540998:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320376.541003:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320376.541006:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985409536:0@lo:38:kworker.0 00000100:00080000:1.0:1767320376.541014:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320376.541017:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320376.541020:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985409664:0@lo:38:kworker.0 00000100:00080000:1.0:1767320376.541026:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320376.541028:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320376.541031:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985409920:0@lo:38:kworker.0 00000100:00080000:2.0:1767320376.541117:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.541119:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.541123:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5619 00000100:00080000:2.0:1767320376.541126:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320376.541130:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320376.541146:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579500 x1853163985410176/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320376.541633:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985409792:0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.541671:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320376.541705:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320376.541711:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985410048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.541731:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320376.541755:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985410432:0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.541777:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320376.541793:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320376.541797:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985410304:0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.541815:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320376.541836:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985409792 00000100:00100000:0.0:1767320376.541840:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985409792 00000100:00100000:0.0:1767320376.541875:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410048 00000100:00100000:0.0:1767320376.541878:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410048 00000100:00100000:0.0:1767320376.541897:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5184 00000100:00100000:0.0:1767320376.541903:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985409792:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.541955:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985409792:12345-0@lo:400:kworker.0 Request processed in 52us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320376.541962:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5184 00000100:00100000:0.0:1767320376.541974:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5185 00000100:00100000:0.0:1767320376.541980:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985410048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.542003:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985410048:12345-0@lo:400:kworker.0 Request processed in 23us (271us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320376.542008:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5185 00000100:00080000:0.0:1767320376.542021:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320376.542043:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985410048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.542051:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985409792:0@lo:400:kworker.0 00000100:00100000:2.0:1767320376.543140:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410432 00000100:00100000:2.0:1767320376.543148:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410432 00000100:00100000:2.0:1767320376.543214:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410304 00000100:00100000:2.0:1767320376.543217:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410304 00000100:00100000:2.0:1767320376.543237:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3375 00000100:00100000:2.0:1767320376.543243:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985410432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320376.543294:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985410432:12345-0@lo:400:kworker.0 Request processed in 51us (1516us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320376.543300:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3375 00000100:00100000:2.0:1767320376.543313:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3376 00000100:00100000:2.0:1767320376.543318:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985410304:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320376.543343:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985410304:12345-0@lo:400:kworker.0 Request processed in 25us (1526us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320376.543349:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3376 00000100:00100000:1.0:1767320376.548656:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985410176:0@lo:38:kworker.0 00000100:00100000:1.0:1767320376.548732:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320376.549676:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985410304:0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.549693:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985410432:0@lo:400:kworker.0 00000100:00100000:3.0:1767320376.550920:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410176 00000100:00100000:3.0:1767320376.550931:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410176 00000100:00100000:3.0:1767320376.550982:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5852 00000100:00100000:3.0:1767320376.550990:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985410176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320376.551011:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320376.551085:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985410176:12345-0@lo:38:kworker.0 Request processed in 95us (2352us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320376.551092:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5852 00000100:00080000:1.0:1767320376.551244:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320376.551249:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320376.551253:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985410176:0@lo:38:kworker.0 00010000:00010000:1.0:1767320377.179861:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320378.204477:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320379.227586:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320380.123463:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985410560:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320380.123530:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320380.123637:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410560 00000100:00100000:1.0:1767320380.123646:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410560 00000100:00100000:1.0:1767320380.123707:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2572 00000100:00100000:1.0:1767320380.123716:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985410560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320380.123743:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320380.124017:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985410688:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320380.124044:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320380.124077:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410688 00000100:00100000:2.0:1767320380.124080:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410688 00000100:00100000:2.0:1767320380.124104:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2573 00000100:00100000:2.0:1767320380.124109:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985410688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320380.124124:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320380.124152:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985410688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 42us (107us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320380.124158:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2573 00000100:00100000:2.0:1767320380.124226:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985410688:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320380.124250:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320380.124254:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320380.125810:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985410560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2089us (2273us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320380.125821:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2572 00000100:00100000:0.0:1767320380.129435:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985410560:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320380.252033:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320381.279378:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320381.610602:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320381.610735:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920613888 00000100:00100000:0.0:1767320381.610746:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920613888 00000100:00100000:0.0:1767320381.610846:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5186 00000100:00100000:0.0:1767320381.610855:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920613888:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320381.610939:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920613888:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (346us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320381.610947:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5186 00000100:00080000:2.0:1767320381.611035:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320381.615078:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320381.615363:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920613760 00000100:00100000:2.0:1767320381.615373:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920613760 00000100:00100000:2.0:1767320381.615424:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5853 00000100:00100000:2.0:1767320381.615431:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920613760:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320381.615452:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320381.615818:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920613760:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 384us (747us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320381.615825:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5853 00000100:00080000:1.0:1767320381.659266:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320381.659276:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320381.659289:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320381.659294:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320381.659299:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320381.659308:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5624 00000100:00080000:1.0:1767320381.659312:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320381.659319:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320381.659357:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4000 x1853163985410816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320381.659478:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320381.659483:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320381.659485:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320381.659489:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5624 00000100:00080000:1.0:1767320381.659491:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320381.659495:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320381.659510:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7480 x1853163985410944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320381.659558:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320381.659563:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320381.659564:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320381.659568:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5624 00000100:00080000:1.0:1767320381.659570:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320381.659573:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320381.659583:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4380 x1853163985411200/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320381.659599:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320381.659603:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320381.659605:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320381.659608:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5624 00000100:00080000:1.0:1767320381.659611:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320381.659614:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320381.659626:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985411328:0@lo:400:kworker.0 00000100:00100000:1.0:1767320381.659635:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985411456/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320381.659692:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985411584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.659699:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320381.659733:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320381.659762:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320381.659763:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320381.659771:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985411072:0@lo:400:kworker.0 00000100:00100000:1.0:1767320381.659774:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985411712:0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.659806:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320381.659813:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320381.659882:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985410816:0@lo:38:kworker.0 00000100:00100000:3.0:1767320381.659967:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411328 00000100:00100000:3.0:1767320381.659975:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411328 00000100:00100000:3.0:1767320381.660028:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411072 00000100:00100000:3.0:1767320381.660032:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411072 00000100:00100000:3.0:1767320381.660056:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5187 00000100:00100000:3.0:1767320381.660065:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985411328:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320381.660427:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320381.660454:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985410944:0@lo:38:kworker.0 00000100:00100000:2.0:1767320381.660479:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320381.660501:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985411200:0@lo:38:kworker.0 00000100:00100000:2.0:1767320381.660517:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320381.660529:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985411456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320381.660548:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320381.660572:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410816 00000100:00100000:2.0:1767320381.660576:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410816 00000100:00100000:2.0:1767320381.660589:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411200 00000100:00100000:2.0:1767320381.660591:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411200 00000100:00100000:2.0:1767320381.660598:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411456 00000100:00100000:2.0:1767320381.660600:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411456 00000100:00100000:2.0:1767320381.660609:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5188 00000100:00100000:2.0:1767320381.660617:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985411072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320381.660805:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985411072:12345-0@lo:400:kworker.0 Request processed in 190us (999us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320381.660810:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5188 00000100:00100000:2.0:1767320381.660834:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5854 00000100:00100000:2.0:1767320381.660837:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985410816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320381.660850:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320381.660870:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985410816:12345-0@lo:38:kworker.0 Request processed in 32us (445us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320381.660874:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5854 00000100:00100000:2.0:1767320381.660882:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5855 00000100:00100000:2.0:1767320381.660885:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985411200:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320381.660890:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320381.660902:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985411200:12345-0@lo:38:kworker.0 Request processed in 16us (384us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320381.660905:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5855 00000100:00100000:2.0:1767320381.660913:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5856 00000100:00100000:2.0:1767320381.660917:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985411456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320381.660924:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320381.660944:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985411456:12345-0@lo:38:kworker.0 Request processed in 25us (394us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320381.660949:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5856 00000100:00100000:2.0:1767320381.660966:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985410944 00000100:00100000:2.0:1767320381.660970:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985410944 00000100:00100000:2.0:1767320381.660991:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1954 00000100:00100000:2.0:1767320381.660995:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985410944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320381.661004:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320381.661027:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985410944:12345-0@lo:38:kworker.0 Request processed in 31us (548us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320381.661031:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1954 00000100:00080000:2.0:1767320381.661049:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320381.661068:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320381.661072:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320381.661074:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985410816:0@lo:38:kworker.0 00000100:00080000:2.0:1767320381.661079:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320381.661081:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320381.661083:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985410944:0@lo:38:kworker.0 00000100:00080000:2.0:1767320381.661087:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320381.661089:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320381.661091:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985411200:0@lo:38:kworker.0 00000100:00080000:2.0:1767320381.661094:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320381.661096:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320381.661097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985411456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320381.661122:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985411072:0@lo:400:kworker.0 00000100:00100000:3.0:1767320381.661249:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985411328:12345-0@lo:400:kworker.0 Request processed in 1185us (1550us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320381.661259:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5187 00000100:00100000:3.0:1767320381.661301:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985411328:0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.662292:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411584 00000100:00100000:0.0:1767320381.662301:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411584 00000100:00100000:2.0:1767320381.662304:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411712 00000100:00100000:2.0:1767320381.662310:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411712 00000100:00100000:0.0:1767320381.662388:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3377 00000100:00100000:0.0:1767320381.662396:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985411584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320381.662446:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3378 00000100:00100000:2.0:1767320381.662455:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985411712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.662512:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985411584:12345-0@lo:400:kworker.0 Request processed in 115us (2779us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320381.662519:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3377 00000100:00100000:2.0:1767320381.662694:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985411712:12345-0@lo:400:kworker.0 Request processed in 240us (2881us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320381.662701:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3378 00000100:00100000:0.0:1767320381.662863:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985411712:0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.662873:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985411584:0@lo:400:kworker.0 00010000:00010000:1.0:1767320382.299449:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320383.323636:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320384.350538:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320385.243551:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985411840:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320385.243638:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985411968:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320385.243645:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320385.243691:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320385.252841:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411840 00000100:00100000:0.0:1767320385.252859:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411840 00000100:00100000:0.0:1767320385.252931:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985411968 00000100:00100000:0.0:1767320385.252936:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985411968 00000100:00100000:0.0:1767320385.252964:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2574 00000100:00100000:0.0:1767320385.252975:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985411840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320385.253009:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320385.253096:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985411840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 123us (9452us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320385.253103:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2574 00000100:00100000:0.0:1767320385.253120:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2575 00000100:00100000:0.0:1767320385.253127:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985411968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320385.253143:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320385.253189:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985411968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (9498us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320385.253195:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2575 00000100:00100000:0.0:1767320385.253283:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985411840:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320385.253629:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985411968:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320385.253786:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320385.253793:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320385.371461:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320386.396434:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320386.730257:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320386.730313:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320386.730417:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920614144 00000100:00100000:3.0:1767320386.730431:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920614144 00000100:00100000:3.0:1767320386.730538:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920614016 00000100:00100000:3.0:1767320386.730543:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920614016 00000100:00100000:3.0:1767320386.730572:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5189 00000100:00100000:3.0:1767320386.730595:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920614144:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320386.730618:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5857 00000100:00100000:0.0:1767320386.730631:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920614016:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320386.730653:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320386.730706:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920614144:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (462us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320386.730718:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5189 00000100:00100000:0.0:1767320386.730726:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920614016:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 90us (407us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320386.730734:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5857 00000100:00080000:1.0:1767320386.731060:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320386.782246:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320386.782261:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320386.782265:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320386.782270:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320386.782275:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320386.782286:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5629 00000100:00080000:1.0:1767320386.782292:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320386.782300:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320386.782362:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca5180 x1853163985412096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320386.782402:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320386.782407:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320386.782409:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320386.782413:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5629 00000100:00080000:1.0:1767320386.782416:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320386.782420:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320386.782438:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6300 x1853163985412224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320386.782498:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320386.782503:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320386.782505:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320386.782509:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5629 00000100:00080000:1.0:1767320386.782512:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320386.782516:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320386.782528:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6680 x1853163985412480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320386.782546:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320386.782549:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320386.782551:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320386.782555:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5629 00000100:00080000:1.0:1767320386.782565:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320386.782569:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320386.782706:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985412352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320386.782790:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320386.782864:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320386.782870:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985412608:0@lo:400:kworker.0 00000100:00100000:0.0:1767320386.782889:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320386.783121:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985412736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320386.783268:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985412864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320386.783303:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320386.783336:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320386.783342:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985412992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320386.783362:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320386.783441:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412864 00000100:00100000:2.0:1767320386.783457:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412864 00000100:00100000:3.0:1767320386.783470:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412352 00000100:00100000:3.0:1767320386.783479:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412352 00000100:00100000:2.0:1767320386.783534:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412992 00000100:00100000:2.0:1767320386.783541:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412992 00000100:00100000:3.0:1767320386.783547:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412608 00000100:00100000:3.0:1767320386.783553:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412608 00000100:00100000:2.0:1767320386.783580:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3379 00000100:00100000:2.0:1767320386.783593:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985412864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320386.783607:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5190 00000100:00100000:3.0:1767320386.783618:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985412352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.783715:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3380 00000100:00100000:1.0:1767320386.783725:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985412992:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320386.783777:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320386.783793:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5191 00000100:00100000:0.0:1767320386.783799:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985412608:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320386.788927:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985412352:12345-0@lo:400:kworker.0 Request processed in 5295us (6119us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320386.789901:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5190 00000100:00100000:3.0:1767320386.790177:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985412096:0@lo:38:kworker.0 00000100:00100000:3.0:1767320386.790281:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320386.790337:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985412224:0@lo:38:kworker.0 00000100:00100000:3.0:1767320386.790373:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320386.792149:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985412480:0@lo:38:kworker.0 00000100:00100000:3.0:1767320386.792240:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320386.792276:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985412736:0@lo:38:kworker.0 00000100:00100000:3.0:1767320386.792300:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320386.792362:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412096 00000100:00100000:3.0:1767320386.792373:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412096 00000100:00100000:3.0:1767320386.792617:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412480 00000100:00100000:3.0:1767320386.792621:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412480 00000100:00100000:3.0:1767320386.792638:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412736 00000100:00100000:3.0:1767320386.792641:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412736 00000100:00100000:3.0:1767320386.792656:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5858 00000100:00100000:3.0:1767320386.792664:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985412096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320386.792693:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320386.792770:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985412096:12345-0@lo:38:kworker.0 Request processed in 98us (2482us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320386.792778:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5858 00000100:00100000:3.0:1767320386.792803:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5859 00000100:00100000:3.0:1767320386.792807:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985412480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320386.792816:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320386.792847:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985412480:12345-0@lo:38:kworker.0 Request processed in 39us (608us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320386.792853:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5859 00000100:00100000:3.0:1767320386.792870:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5860 00000100:00100000:3.0:1767320386.792874:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985412736:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320386.792892:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320386.792925:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985412736:12345-0@lo:38:kworker.0 Request processed in 48us (623us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320386.792931:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5860 00000100:00100000:2.0:1767320386.794270:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985412864:12345-0@lo:400:kworker.0 Request processed in 10669us (10954us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320386.794919:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985412352:0@lo:400:kworker.0 00000100:00080000:3.0:1767320386.795019:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320386.795032:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320386.795036:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985412096:0@lo:38:kworker.0 00000100:00080000:3.0:1767320386.795057:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320386.795061:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320386.795064:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985412480:0@lo:38:kworker.0 00000100:00080000:3.0:1767320386.795082:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320386.795086:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320386.795089:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985412736:0@lo:38:kworker.0 00000100:00100000:2.0:1767320386.795150:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3379 00000100:00100000:3.0:1767320386.795248:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985412224 00000100:00100000:3.0:1767320386.795260:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985412224 00000100:00100000:2.0:1767320386.795852:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985412864:0@lo:400:kworker.0 00000100:00100000:3.0:1767320386.795894:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1955 00000100:00100000:3.0:1767320386.795905:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985412224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320386.795933:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320386.796047:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985412224:12345-0@lo:38:kworker.0 Request processed in 141us (5671us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320386.796055:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1955 00000100:00080000:2.0:1767320386.796508:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320386.796517:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320386.796522:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985412224:0@lo:38:kworker.0 00000100:00100000:1.0:1767320386.801273:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985412992:12345-0@lo:400:kworker.0 Request processed in 17530us (17889us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320386.801293:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3380 00000100:00100000:3.0:1767320386.802562:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985412992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320386.805197:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985412608:12345-0@lo:400:kworker.0 Request processed in 21348us (22256us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320386.805223:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5191 00000100:00100000:3.0:1767320386.808265:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985412608:0@lo:400:kworker.0 00010000:00010000:2.0:1767320387.427482:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320388.446038:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320389.468652:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320390.365621:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320390.365657:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985413120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320390.365753:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320390.366987:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985413120 00000100:00100000:0.0:1767320390.367006:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985413120 00000100:00080000:1.0:1767320390.367093:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320390.367110:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2576 00000100:00100000:0.0:1767320390.367133:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985413120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320390.368508:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320390.368813:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985413248:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320390.368869:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320390.372059:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985413248 00000100:00100000:3.0:1767320390.372081:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985413248 00000100:00100000:3.0:1767320390.372227:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2577 00000100:00100000:3.0:1767320390.372240:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985413248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320390.372287:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767320390.372524:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320390.384547:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985413248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 12295us (15663us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320390.384565:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2577 00000100:00100000:0.0:1767320390.386312:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985413120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 19179us (20545us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320390.386330:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2576 00000100:00100000:2.0:1767320390.392595:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985413120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320390.392660:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985413248:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320390.491369:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320391.518544:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767320391.851392:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920614272 offset 0 length 520: 4 00000100:00100000:3.0:1767320391.851442:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320391.851485:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320391.851610:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920614528 00000100:00100000:2.0:1767320391.851620:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920614528 00000100:00100000:2.0:1767320391.851702:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920614400 00000100:00100000:2.0:1767320391.851707:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920614400 00000100:00100000:2.0:1767320391.851727:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5192 00000100:00100000:2.0:1767320391.851740:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920614528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320391.851824:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920614528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (381us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320391.851833:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5192 00000100:00100000:2.0:1767320391.851853:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5861 00000100:00100000:2.0:1767320391.851858:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920614400:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320391.851878:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320391.851908:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920614400:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (422us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320391.851914:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5861 00000100:00080000:1.0:1767320391.853639:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320391.899482:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320366/real 1767320366] req@ffff8ce184a11c00 x1853163985406848/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320391 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320391.899518:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce184a11c00 x1853163985406848/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320391 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320391.899538:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320391.899563:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320391.899567:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985406848:0@lo:250:kworker.0 00000100:00080000:2.0:1767320391.899690:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320391.899696:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320391.899699:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320391.899704:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320391.899709:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320391.899715:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320391.899748:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a680 x1853163985413376/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320391.899779:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320391.899783:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320391.899785:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320391.899789:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5635 00000100:00080000:2.0:1767320391.899792:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320391.899795:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320391.899810:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b480 x1853163985413504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320391.899820:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320391.899823:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320391.899825:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320391.899828:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5635 00000100:00080000:2.0:1767320391.899831:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320391.899834:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320391.899846:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578e00 x1853163985413632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320391.899876:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320391.899880:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320391.899881:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320391.899885:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5635 00000100:00080000:2.0:1767320391.899888:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320391.899891:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320391.899902:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4700 x1853163985413888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320391.899918:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320391.899921:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320391.899923:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320391.899926:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5635 00000100:00080000:2.0:1767320391.899928:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320391.899931:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320391.899941:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5880 x1853163985414144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320391.899994:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985414400:0@lo:400:kworker.0 00000100:00100000:3.0:1767320391.900015:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985413760:0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.900071:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320391.900072:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320391.900153:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320391.900161:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985414016:0@lo:400:kworker.0 00000100:00100000:3.0:1767320391.900202:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320391.900206:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985413376:0@lo:250:kworker.0 00000100:00100000:3.0:1767320391.900234:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985413760 00000400:00000100:0.0:1767320391.900240:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985413376 offset 0 length 520: 4 00000100:00100000:3.0:1767320391.900243:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985413760 00000100:00100000:0.0:1767320391.900266:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985413504:0@lo:38:kworker.0 00000100:00100000:0.0:1767320391.900291:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320391.900291:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414016 00000100:00100000:3.0:1767320391.900296:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414016 00000100:00100000:0.0:1767320391.900316:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985413632:0@lo:38:kworker.0 00000100:00100000:3.0:1767320391.900319:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985413504 00000100:00100000:3.0:1767320391.900323:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985413504 00000100:00100000:3.0:1767320391.900347:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5193 00000100:00100000:0.0:1767320391.900348:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320391.900359:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985413760:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320391.900372:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985413888:0@lo:38:kworker.0 00000100:00100000:0.0:1767320391.900392:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320391.900405:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985414144:0@lo:38:kworker.0 00000100:00100000:3.0:1767320391.900434:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985413760:12345-0@lo:400:kworker.0 Request processed in 75us (363us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320391.900436:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320391.900465:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985413888 00000100:00100000:0.0:1767320391.900470:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985413888 00000100:00100000:0.0:1767320391.900484:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414144 00000100:00100000:0.0:1767320391.900486:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414144 00000100:00100000:0.0:1767320391.900495:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5194 00000100:00100000:0.0:1767320391.900502:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985414016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320391.900548:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985414016:12345-0@lo:400:kworker.0 Request processed in 46us (345us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320391.900555:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5194 00000100:00100000:0.0:1767320391.900573:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5862 00000100:00100000:0.0:1767320391.900577:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985413504:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320391.900595:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320391.900626:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985413504:12345-0@lo:38:kworker.0 Request processed in 47us (336us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320391.900632:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5862 00000100:00100000:3.0:1767320391.900885:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5193 00000100:00100000:3.0:1767320391.900902:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5863 00000100:00100000:3.0:1767320391.900907:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985413888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320391.900922:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320391.900923:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985414272:0@lo:400:kworker.0 00000100:00100000:3.0:1767320391.900954:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985413888:12345-0@lo:38:kworker.0 Request processed in 45us (560us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320391.900962:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5863 00000100:00100000:3.0:1767320391.900979:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5864 00000100:00100000:3.0:1767320391.900983:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985414144:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320391.900993:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320391.901019:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985414144:12345-0@lo:38:kworker.0 Request processed in 34us (584us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320391.901024:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5864 00000100:00100000:3.0:1767320391.901066:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985413632 00000100:00100000:3.0:1767320391.901071:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985413632 00000100:00100000:3.0:1767320391.901104:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1956 00000100:00100000:3.0:1767320391.901109:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985413632:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320391.901121:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320391.901147:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985413632:12345-0@lo:38:kworker.0 Request processed in 37us (803us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320391.901153:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1956 00000100:00100000:3.0:1767320391.901213:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985414016:0@lo:400:kworker.0 00000100:00100000:3.0:1767320391.901223:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985413760:0@lo:400:kworker.0 00000100:00100000:1.0:1767320391.901287:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320391.901315:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320391.901326:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320391.901333:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985413504:0@lo:38:kworker.0 00000100:00080000:3.0:1767320391.901353:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320391.901359:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320391.901363:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985413632:0@lo:38:kworker.0 00000100:00100000:1.0:1767320391.901379:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414400 00000100:00100000:1.0:1767320391.901388:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414400 00000100:00080000:3.0:1767320391.901396:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320391.901402:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320391.901406:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985413888:0@lo:38:kworker.0 00000100:00080000:3.0:1767320391.901426:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320391.901431:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320391.901436:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985414144:0@lo:38:kworker.0 00000100:00100000:1.0:1767320391.901443:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414272 00000100:00100000:1.0:1767320391.901449:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414272 00000100:00100000:1.0:1767320391.901474:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3381 00000100:00100000:1.0:1767320391.901483:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985414400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320391.902246:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3382 00000100:00100000:3.0:1767320391.902262:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985414272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320391.902330:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985414272:12345-0@lo:400:kworker.0 Request processed in 75us (1048us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320391.902340:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3382 00000100:00100000:1.0:1767320391.904002:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985414400:12345-0@lo:400:kworker.0 Request processed in 2508us (3922us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320391.904021:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3381 00000100:00080000:3.0:1767320391.904119:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320391.904134:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985414272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.905823:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985414400:0@lo:400:kworker.0 00010000:00010000:0.0:1767320392.540204:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320393.563419:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320394.587380:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320395.483847:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985414528:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320395.483933:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320395.483938:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985414656:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320395.483985:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320395.484048:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414528 00000100:00100000:2.0:1767320395.484056:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414528 00000100:00100000:2.0:1767320395.484103:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414656 00000100:00100000:2.0:1767320395.484107:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414656 00000100:00100000:2.0:1767320395.484129:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2578 00000100:00100000:2.0:1767320395.484139:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985414528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320395.484196:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320395.484209:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2579 00000100:00100000:1.0:1767320395.484222:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985414656:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320395.484241:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320395.484276:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985414656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (290us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320395.484284:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2579 00000100:00100000:1.0:1767320395.484480:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985414656:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320395.484527:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320395.484531:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320395.485841:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985414528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1699us (1904us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320395.485851:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2578 00000100:00100000:3.0:1767320395.486273:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985414528:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320395.613507:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320396.639578:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320396.970721:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320396.970810:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320396.976434:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920614784 00000100:00100000:0.0:1767320396.976461:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920614784 00000100:00100000:0.0:1767320396.976635:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920614656 00000100:00100000:0.0:1767320396.976640:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920614656 00000100:00100000:0.0:1767320396.976679:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5195 00000100:00100000:0.0:1767320396.976699:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920614784:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320396.976724:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5865 00000100:00100000:3.0:1767320396.976737:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920614656:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320396.976763:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320396.976814:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920614784:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (6100us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320396.976824:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5195 00000100:00100000:3.0:1767320396.976921:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920614656:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 180us (6107us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320396.976932:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5865 00000100:00080000:1.0:1767320396.980374:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320397.022127:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320397.022137:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320397.022140:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320397.022145:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320397.022149:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320397.022159:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320397.022195:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320397.022203:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320397.022244:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7800 x1853163985414784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320397.022272:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320397.022276:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320397.022277:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320397.022281:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320397.022283:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320397.022287:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320397.022298:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6d80 x1853163985414912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320397.022332:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320397.022336:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320397.022338:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320397.022341:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320397.022343:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320397.022346:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320397.022356:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4e00 x1853163985415168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320397.022373:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320397.022377:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320397.022379:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320397.022383:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320397.022385:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320397.022389:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320397.022398:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4380 x1853163985415424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320397.022431:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985415040:0@lo:400:kworker.0 00000100:00100000:2.0:1767320397.022467:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985415552:0@lo:400:kworker.0 00000100:00100000:3.0:1767320397.022523:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985414784:0@lo:38:kworker.0 00000100:00100000:0.0:1767320397.022577:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985415296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320397.029883:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320397.029963:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320397.029975:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985415680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320397.030029:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320397.030108:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415552 00000100:00100000:2.0:1767320397.030115:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415552 00000100:00100000:2.0:1767320397.030186:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415680 00000100:00100000:2.0:1767320397.030190:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415680 00000100:00100000:2.0:1767320397.030214:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3383 00000100:00100000:2.0:1767320397.030222:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985415552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320397.030275:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985415552:12345-0@lo:400:kworker.0 Request processed in 52us (398us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320397.030282:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3383 00000100:00100000:2.0:1767320397.030299:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3384 00000100:00100000:2.0:1767320397.030304:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985415680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320397.030330:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985415680:12345-0@lo:400:kworker.0 Request processed in 26us (300us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320397.030336:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3384 00000100:00080000:2.0:1767320397.030354:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320397.030357:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320397.030382:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985415680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320397.030393:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985415552:0@lo:400:kworker.0 00000100:00100000:1.0:1767320397.031049:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320397.031130:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415040 00000100:00100000:1.0:1767320397.031140:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415040 00000100:00100000:1.0:1767320397.031232:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5196 00000100:00100000:1.0:1767320397.031244:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985415040:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320397.031313:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985415040:12345-0@lo:400:kworker.0 Request processed in 72us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320397.031320:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5196 00000100:00080000:1.0:1767320397.031346:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320397.031395:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985415040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320397.036115:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320397.036265:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985414912:0@lo:38:kworker.0 00000100:00100000:3.0:1767320397.036319:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320397.036361:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985415168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320397.036384:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320397.038724:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414912 00000100:00100000:1.0:1767320397.038737:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414912 00000100:00100000:1.0:1767320397.038787:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1957 00000100:00100000:1.0:1767320397.038794:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985414912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320397.038817:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320397.038881:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985414912:12345-0@lo:38:kworker.0 Request processed in 87us (2562us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320397.038887:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1957 00000100:00100000:3.0:1767320397.041630:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985415424:0@lo:38:kworker.0 00000100:00100000:3.0:1767320397.041686:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320397.043757:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985414784 00000100:00100000:2.0:1767320397.043771:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985414784 00000100:00100000:2.0:1767320397.043839:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415168 00000100:00100000:2.0:1767320397.043842:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415168 00000100:00100000:2.0:1767320397.043872:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5866 00000100:00100000:2.0:1767320397.043880:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985414784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320397.043904:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320397.043976:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985414784:12345-0@lo:38:kworker.0 Request processed in 95us (7868us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320397.043984:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5866 00000100:00100000:2.0:1767320397.044012:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5867 00000100:00100000:2.0:1767320397.044017:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985415168:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320397.044027:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320397.044052:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985415168:12345-0@lo:38:kworker.0 Request processed in 33us (7666us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320397.044057:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5867 00000100:00080000:3.0:1767320397.045866:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320397.045880:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320397.045886:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985414784:0@lo:38:kworker.0 00000100:00080000:3.0:1767320397.045899:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320397.045902:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320397.045905:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985414912:0@lo:38:kworker.0 00000100:00080000:3.0:1767320397.045914:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320397.045918:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320397.045921:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985415168:0@lo:38:kworker.0 00000100:00100000:1.0:1767320397.047499:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415424 00000100:00100000:1.0:1767320397.047512:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415424 00000100:00100000:2.0:1767320397.047600:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320397.054597:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415296 00000100:00100000:0.0:1767320397.054624:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415296 00000100:00100000:0.0:1767320397.054715:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5197 00000100:00100000:0.0:1767320397.054725:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985415296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320397.054809:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985415296:12345-0@lo:400:kworker.0 Request processed in 86us (7212us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320397.054816:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5197 00000100:00100000:0.0:1767320397.054851:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5868 00000100:00100000:0.0:1767320397.054855:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985415424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320397.054874:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320397.054910:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985415424:12345-0@lo:38:kworker.0 Request processed in 54us (13224us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320397.054916:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5868 00000100:00080000:0.0:1767320397.054953:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320397.055017:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985415296:0@lo:400:kworker.0 00000100:00080000:3.0:1767320397.056783:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320397.056794:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320397.056799:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985415424:0@lo:38:kworker.0 00010000:00010000:1.0:1767320397.660329:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320398.683397:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320399.716151:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320400.603558:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320400.603582:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985415808:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320400.603649:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320400.606481:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415808 00000100:00100000:3.0:1767320400.606504:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415808 00000100:00100000:3.0:1767320400.606604:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2580 00000100:00100000:3.0:1767320400.606617:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985415808:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320400.606635:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320400.606656:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985415936:0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320400.606656:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767320400.606753:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320400.613848:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320400.620276:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985415808:12345-0@lo:13:osp-pre-0-1.0 Request processed in 13653us (16616us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320400.620290:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2580 00000100:00100000:3.0:1767320400.620331:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985415936 00000100:00100000:3.0:1767320400.620341:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985415936 00000100:00100000:3.0:1767320400.620409:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2581 00000100:00100000:3.0:1767320400.620417:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985415936:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320400.620451:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320400.620498:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985415936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (6658us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320400.620508:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2581 00000100:00100000:0.0:1767320400.620592:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985415808:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320400.620753:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320400.620785:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985415936:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320400.735530:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320401.757125:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320402.089806:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320402.089935:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320402.090317:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920615040 00000100:00100000:2.0:1767320402.090328:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920615040 00000100:00100000:2.0:1767320402.090406:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920614912 00000100:00100000:2.0:1767320402.090410:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920614912 00000100:00100000:2.0:1767320402.090439:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5198 00000100:00080000:1.0:1767320402.090474:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320402.090525:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920615040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320402.090610:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920615040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (814us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320402.091584:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5869 00000100:00100000:0.0:1767320402.091594:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920614912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320402.091619:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320402.092051:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5198 00000100:00100000:1.0:1767320402.093011:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920614912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1410us (3069us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320402.093021:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5869 00000100:00080000:0.0:1767320402.139362:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320402.139373:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320402.139376:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320402.139380:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320402.139384:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320402.139391:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5645 00000100:00080000:0.0:1767320402.139395:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320402.139401:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320402.139444:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac000 x1853163985416064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320402.139467:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320402.139471:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320402.139473:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320402.139475:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5645 00000100:00080000:0.0:1767320402.139477:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320402.139480:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320402.139490:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad180 x1853163985416192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320402.139573:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320402.139577:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320402.139580:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320402.139584:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5645 00000100:00080000:0.0:1767320402.139586:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320402.139590:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320402.139612:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac380 x1853163985416448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320402.139625:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985416064:0@lo:38:kworker.0 00000100:00100000:1.0:1767320402.139688:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985416576:0@lo:400:kworker.0 00000100:00100000:1.0:1767320402.139761:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320402.139823:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320402.139830:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985416320:0@lo:400:kworker.0 00000100:00100000:1.0:1767320402.139853:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320402.139890:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416576 00000100:00100000:1.0:1767320402.139898:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416576 00000100:00100000:1.0:1767320402.139982:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416320 00000100:00100000:1.0:1767320402.139986:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416320 00000100:00100000:1.0:1767320402.140006:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5199 00000100:00100000:1.0:1767320402.140014:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985416576:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320402.140062:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985416576:12345-0@lo:400:kworker.0 Request processed in 49us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320402.140069:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5199 00000100:00100000:1.0:1767320402.140084:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5200 00000100:00100000:1.0:1767320402.140090:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985416320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320402.140116:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985416320:12345-0@lo:400:kworker.0 Request processed in 27us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320402.140122:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5200 00000100:00080000:1.0:1767320402.140137:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320402.140192:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985416320:0@lo:400:kworker.0 00000100:00100000:1.0:1767320402.140205:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985416576:0@lo:400:kworker.0 00000100:00100000:3.0:1767320402.140897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320402.140948:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985416192:0@lo:38:kworker.0 00000100:00100000:3.0:1767320402.140970:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320402.140992:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985416448:0@lo:38:kworker.0 00000100:00100000:3.0:1767320402.141016:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320402.141048:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416064 00000100:00100000:3.0:1767320402.141053:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416064 00000100:00100000:3.0:1767320402.141077:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416448 00000100:00100000:3.0:1767320402.141080:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416448 00000100:00100000:3.0:1767320402.141095:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5870 00000100:00100000:3.0:1767320402.141099:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985416064:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320402.141118:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320402.141147:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985416064:12345-0@lo:38:kworker.0 Request processed in 46us (252us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320402.141152:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5870 00000100:00100000:3.0:1767320402.141175:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5871 00000100:00100000:3.0:1767320402.141178:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985416448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320402.141185:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320402.141214:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985416448:12345-0@lo:38:kworker.0 Request processed in 34us (197us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320402.141220:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5871 00000100:00080000:3.0:1767320402.141251:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320402.141257:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320402.141261:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985416448:0@lo:38:kworker.0 00000100:00080000:3.0:1767320402.141270:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320402.141273:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320402.141275:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985416064:0@lo:38:kworker.0 00000100:00100000:3.0:1767320402.141302:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416192 00000100:00100000:3.0:1767320402.141305:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416192 00000100:00100000:3.0:1767320402.141318:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1958 00000100:00100000:3.0:1767320402.141321:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985416192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320402.141327:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320402.141345:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985416192:12345-0@lo:38:kworker.0 Request processed in 24us (375us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320402.141349:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1958 00000100:00080000:3.0:1767320402.141363:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320402.141366:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320402.141368:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985416192:0@lo:38:kworker.0 00000100:00080000:0.0:1767320402.144285:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320402.144298:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320402.144302:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320402.144310:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5645 00000100:00080000:0.0:1767320402.144316:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320402.144322:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320402.144366:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886aea00 x1853163985416704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320402.144445:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985416832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320402.144464:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985416960:0@lo:400:kworker.0 00000100:00100000:3.0:1767320402.144930:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985416704:0@lo:38:kworker.0 00000100:00100000:1.0:1767320402.145109:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320402.145313:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416832 00000100:00100000:2.0:1767320402.145322:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416832 00000100:00100000:2.0:1767320402.145385:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3385 00000100:00100000:2.0:1767320402.145454:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985416832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320402.145515:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985416832:12345-0@lo:400:kworker.0 Request processed in 66us (407us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320402.145522:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3385 00000100:00080000:2.0:1767320402.145546:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320402.145620:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985416832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320402.151900:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320402.152042:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416960 00000100:00100000:2.0:1767320402.152053:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416960 00000100:00100000:2.0:1767320402.152122:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3386 00000100:00100000:2.0:1767320402.152133:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985416960:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320402.152243:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985416960:12345-0@lo:400:kworker.0 Request processed in 113us (351us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320402.152250:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3386 00000100:00080000:1.0:1767320402.152548:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320402.152992:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320402.153732:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985416704 00000100:00100000:1.0:1767320402.153742:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985416704 00000100:00100000:1.0:1767320402.154027:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5872 00000100:00100000:1.0:1767320402.154034:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985416704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320402.154057:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320402.154129:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985416704:12345-0@lo:38:kworker.0 Request processed in 94us (1138us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320402.154135:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5872 00000100:00080000:2.0:1767320402.154224:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320402.154231:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320402.154235:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985416704:0@lo:38:kworker.0 00000100:00100000:0.0:1767320402.154464:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985416960:0@lo:400:kworker.0 00010000:00010000:1.0:1767320402.779552:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320403.803945:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320404.833472:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320405.723451:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320405.723480:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985417088:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320405.723549:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320405.723672:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417088 00000100:00100000:1.0:1767320405.723682:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417088 00000100:00080000:0.0:1767320405.723727:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320405.723746:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2582 00000100:00100000:1.0:1767320405.723755:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985417088:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320405.723791:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320405.723883:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985417216:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320405.723917:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320405.723978:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985417088:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320405.724006:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417216 00000100:00100000:3.0:1767320405.724010:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417216 00000100:00100000:3.0:1767320405.724040:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2583 00000100:00100000:3.0:1767320405.724047:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985417216:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320405.724063:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767320405.724090:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320405.724100:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985417216:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (181us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320405.724107:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2583 00000100:00100000:2.0:1767320405.724369:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985417216:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320405.724916:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985417088:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (287us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320405.724938:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2582 00010000:00010000:2.0:1767320405.851468:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320406.875453:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320407.210480:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320407.210576:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320407.210764:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920615296 00000100:00100000:0.0:1767320407.210774:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920615296 00000100:00100000:0.0:1767320407.210857:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920615168 00000100:00100000:0.0:1767320407.210861:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920615168 00000100:00100000:0.0:1767320407.210878:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5201 00000100:00100000:0.0:1767320407.210889:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920615296:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320407.210960:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920615296:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 73us (490us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320407.210967:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5201 00000100:00100000:0.0:1767320407.210984:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5873 00000100:00100000:0.0:1767320407.210988:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920615168:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320407.211025:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320407.211049:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920615168:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 59us (473us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320407.211056:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5873 00000100:00080000:1.0:1767320407.212589:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320407.259450:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320407.259462:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320407.259465:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320407.259469:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320407.259473:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320407.259483:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5650 00000100:00080000:2.0:1767320407.259488:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320407.259495:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320407.259534:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4700 x1853163985417344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320407.259562:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320407.259565:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320407.259568:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320407.259571:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5650 00000100:00080000:2.0:1767320407.259573:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320407.259576:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320407.259590:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4a80 x1853163985417472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320407.259626:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320407.259630:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320407.259632:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320407.259636:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5650 00000100:00080000:2.0:1767320407.259638:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320407.259641:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320407.259651:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4380 x1853163985417728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320407.259666:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320407.259670:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320407.259672:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320407.259674:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5650 00000100:00080000:2.0:1767320407.259676:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320407.259679:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320407.259688:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4e00 x1853163985417984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320407.259727:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985417856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320407.259783:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320407.259848:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320407.259854:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985417600:0@lo:400:kworker.0 00000100:00100000:2.0:1767320407.259886:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320407.259908:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985418240:0@lo:400:kworker.0 00000100:00100000:2.0:1767320407.259931:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320407.259948:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320407.259952:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985418112:0@lo:400:kworker.0 00000100:00100000:2.0:1767320407.259969:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320407.259995:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985418240 00000100:00100000:2.0:1767320407.260002:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985418240 00000100:00100000:2.0:1767320407.260061:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985418112 00000100:00100000:2.0:1767320407.260065:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985418112 00000100:00100000:2.0:1767320407.260088:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3387 00000100:00100000:2.0:1767320407.260099:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985418240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320407.260230:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3388 00000100:00100000:0.0:1767320407.260240:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985418112:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320407.260287:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985418112:12345-0@lo:400:kworker.0 Request processed in 49us (315us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320407.260294:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3388 00000100:00100000:0.0:1767320407.260319:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985417344:0@lo:38:kworker.0 00000100:00100000:0.0:1767320407.260342:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320407.260365:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985417472:0@lo:38:kworker.0 00000100:00100000:0.0:1767320407.260388:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320407.260404:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985417728:0@lo:38:kworker.0 00000100:00100000:0.0:1767320407.260419:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320407.260433:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985417984:0@lo:38:kworker.0 00000100:00100000:0.0:1767320407.260451:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320407.260477:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417856 00000100:00100000:0.0:1767320407.260482:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417856 00000100:00100000:0.0:1767320407.260508:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417600 00000100:00100000:0.0:1767320407.260511:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417600 00000100:00100000:0.0:1767320407.260521:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417344 00000100:00100000:0.0:1767320407.260523:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417344 00000100:00100000:0.0:1767320407.260531:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417728 00000100:00100000:0.0:1767320407.260533:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417728 00000100:00100000:0.0:1767320407.260540:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5202 00000100:00100000:0.0:1767320407.260545:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985417856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320407.260580:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985417856:12345-0@lo:400:kworker.0 Request processed in 34us (797us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320407.260586:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5202 00000100:00100000:0.0:1767320407.260597:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417984 00000100:00100000:0.0:1767320407.260600:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417984 00000100:00100000:0.0:1767320407.260610:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5203 00000100:00100000:0.0:1767320407.260616:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985417600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320407.260642:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985417600:12345-0@lo:400:kworker.0 Request processed in 27us (756us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320407.260648:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5203 00000100:00100000:0.0:1767320407.260659:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5874 00000100:00100000:0.0:1767320407.260663:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985417344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320407.260680:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320407.260708:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985417344:12345-0@lo:38:kworker.0 Request processed in 44us (365us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320407.260714:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5874 00000100:00100000:0.0:1767320407.260724:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5875 00000100:00100000:0.0:1767320407.260727:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985417728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320407.260734:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320407.260754:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985417728:12345-0@lo:38:kworker.0 Request processed in 25us (333us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320407.260759:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5875 00000100:00100000:0.0:1767320407.260768:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5876 00000100:00100000:0.0:1767320407.260772:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985417984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320407.260779:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320407.260797:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985417984:12345-0@lo:38:kworker.0 Request processed in 23us (346us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320407.260802:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5876 00000100:00080000:0.0:1767320407.260845:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320407.260849:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320407.260853:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985417344:0@lo:38:kworker.0 00000100:00080000:0.0:1767320407.260860:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320407.260863:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320407.260866:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985417728:0@lo:38:kworker.0 00000100:00080000:0.0:1767320407.260872:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320407.260874:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320407.260877:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985417984:0@lo:38:kworker.0 00000100:00080000:0.0:1767320407.260922:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320407.260926:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320407.260928:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320407.260957:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985418112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320407.260985:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985417600:0@lo:400:kworker.0 00000100:00100000:0.0:1767320407.260993:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985417856:0@lo:400:kworker.0 00000100:00100000:3.0:1767320407.261230:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985417472 00000100:00100000:3.0:1767320407.261235:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985417472 00000100:00100000:3.0:1767320407.261265:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1959 00000100:00100000:3.0:1767320407.261270:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190dbac50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985417472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320407.261280:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320407.261312:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190dbac50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985417472:12345-0@lo:38:kworker.0 Request processed in 42us (923us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320407.261318:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1959 00000100:00080000:3.0:1767320407.261509:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320407.261513:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320407.261516:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985417472:0@lo:38:kworker.0 00000100:00100000:2.0:1767320407.261739:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985418240:12345-0@lo:400:kworker.0 Request processed in 1643us (1806us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320407.261746:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3387 00000100:00100000:3.0:1767320407.261887:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985418240:0@lo:400:kworker.0 00010000:00010000:1.0:1767320407.899434:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320408.923541:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320409.947650:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320410.843563:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985418368:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320410.843637:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320410.843873:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985418368 00000100:00100000:3.0:1767320410.843881:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985418368 00000100:00100000:3.0:1767320410.843940:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2584 00000100:00100000:3.0:1767320410.843950:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985418368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320410.843978:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767320410.844122:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320410.844334:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985418368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 385us (696us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320410.844345:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2584 00000100:00100000:3.0:1767320410.844393:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985418368:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320410.844552:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985418496:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320410.844583:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320410.845192:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985418496 00000100:00100000:1.0:1767320410.845199:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985418496 00000100:00100000:1.0:1767320410.846279:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2585 00000100:00100000:1.0:1767320410.846292:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985418496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320410.846324:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320410.846403:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985418496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (1817us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320410.846410:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2585 00000100:00100000:3.0:1767320410.846592:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985418496:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320410.846818:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767320410.972865:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320411.995392:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320412.340294:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320412.340346:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320412.340458:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920615552 00000100:00100000:3.0:1767320412.340468:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920615552 00000100:00100000:3.0:1767320412.340541:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920615424 00000100:00100000:3.0:1767320412.340544:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920615424 00000100:00100000:3.0:1767320412.340562:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5204 00000100:00100000:3.0:1767320412.340581:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920615552:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320412.340657:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920615552:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (373us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320412.340665:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5204 00000100:00100000:3.0:1767320412.340684:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5877 00000100:00100000:3.0:1767320412.340688:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920615424:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320412.340707:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320412.340884:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920615424:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 193us (536us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320412.340890:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5877 00000100:00080000:0.0:1767320412.342148:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320412.380630:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320412.380639:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320412.380641:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320412.380646:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320412.380650:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320412.380659:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5655 00000100:00080000:3.0:1767320412.380663:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320412.380670:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320412.380701:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b800 x1853163985418624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320412.380740:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320412.380743:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320412.380745:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320412.380749:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5655 00000100:00080000:3.0:1767320412.380751:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320412.380755:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320412.380768:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579f80 x1853163985418752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320412.380794:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320412.380797:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320412.380798:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320412.380802:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5655 00000100:00080000:3.0:1767320412.380804:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320412.380807:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320412.380818:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7100 x1853163985419008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320412.380834:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320412.380836:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320412.380838:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320412.380841:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5655 00000100:00080000:3.0:1767320412.380843:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320412.380846:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320412.380856:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8bb80 x1853163985419264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320412.380954:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985418880:0@lo:400:kworker.0 00000100:00100000:3.0:1767320412.381014:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320412.381069:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320412.381074:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985419136:0@lo:400:kworker.0 00000100:00100000:3.0:1767320412.381094:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320412.381117:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985419520:0@lo:400:kworker.0 00000100:00100000:3.0:1767320412.381138:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320412.381152:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320412.381157:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985419392:0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.381409:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985419520 00000100:00100000:2.0:1767320412.381416:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985419520 00000100:00100000:2.0:1767320412.381467:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3389 00000100:00100000:2.0:1767320412.381477:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985419520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.381522:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985419520:12345-0@lo:400:kworker.0 Request processed in 47us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320412.381529:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3389 00000100:00100000:2.0:1767320412.381552:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985418880 00000100:00100000:2.0:1767320412.381555:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985418880 00000100:00100000:2.0:1767320412.381579:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985419136 00000100:00100000:2.0:1767320412.381581:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985419136 00000100:00100000:2.0:1767320412.381598:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5205 00000100:00100000:2.0:1767320412.381603:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985418880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.381644:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985418880:12345-0@lo:400:kworker.0 Request processed in 41us (630us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320412.381649:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5205 00000100:00100000:2.0:1767320412.381658:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5206 00000100:00100000:2.0:1767320412.381663:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985419136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.381680:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985419136:12345-0@lo:400:kworker.0 Request processed in 18us (586us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320412.381684:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5206 00000100:00080000:2.0:1767320412.381698:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320412.381701:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320412.381739:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985419136:0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.381747:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985418880:0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.381771:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985418624:0@lo:38:kworker.0 00000100:00100000:2.0:1767320412.381791:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320412.381808:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985418752:0@lo:38:kworker.0 00000100:00100000:2.0:1767320412.381826:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320412.381844:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985419008:0@lo:38:kworker.0 00000100:00100000:2.0:1767320412.381862:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320412.381875:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985419264:0@lo:38:kworker.0 00000100:00100000:2.0:1767320412.381894:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320412.381919:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985418624 00000100:00100000:2.0:1767320412.381922:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985418624 00000100:00100000:2.0:1767320412.381938:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985419008 00000100:00100000:2.0:1767320412.381940:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985419008 00000100:00100000:2.0:1767320412.381949:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985419264 00000100:00100000:2.0:1767320412.381952:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985419264 00000100:00100000:2.0:1767320412.381960:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5878 00000100:00100000:2.0:1767320412.381964:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985418624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320412.381981:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320412.382010:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985418624:12345-0@lo:38:kworker.0 Request processed in 44us (217us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320412.382016:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5878 00000100:00100000:2.0:1767320412.382027:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5879 00000100:00100000:2.0:1767320412.382030:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985419008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320412.382039:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320412.382060:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985419008:12345-0@lo:38:kworker.0 Request processed in 28us (197us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320412.382065:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5879 00000100:00100000:2.0:1767320412.382075:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5880 00000100:00100000:2.0:1767320412.382078:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985419264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320412.382086:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320412.382104:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985419264:12345-0@lo:38:kworker.0 Request processed in 24us (209us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320412.382110:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5880 00000100:00080000:2.0:1767320412.382134:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320412.382138:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320412.382141:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985418624:0@lo:38:kworker.0 00000100:00080000:2.0:1767320412.382149:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320412.382151:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320412.382154:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985419008:0@lo:38:kworker.0 00000100:00080000:2.0:1767320412.382160:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320412.382403:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320412.382407:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985419264:0@lo:38:kworker.0 00000100:00100000:3.0:1767320412.383047:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320412.383133:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985419520:0@lo:400:kworker.0 00000100:00100000:3.0:1767320412.383995:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985418752 00000100:00100000:3.0:1767320412.384003:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985418752 00000100:00100000:3.0:1767320412.384035:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1960 00000100:00100000:3.0:1767320412.384041:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985418752:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320412.384056:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320412.384097:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985418752:12345-0@lo:38:kworker.0 Request processed in 55us (2270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320412.384103:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1960 00000100:00080000:2.0:1767320412.384562:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320412.384578:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320412.384582:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985418752:0@lo:38:kworker.0 00000100:00100000:1.0:1767320412.386328:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985419392 00000100:00100000:1.0:1767320412.386340:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985419392 00000100:00100000:1.0:1767320412.386412:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3390 00000100:00100000:1.0:1767320412.386422:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985419392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320412.386507:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985419392:12345-0@lo:400:kworker.0 Request processed in 87us (3462us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320412.386515:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3390 00000100:00100000:1.0:1767320412.386586:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985419392:0@lo:400:kworker.0 00000100:00080000:2.0:1767320412.386954:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320413.019329:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320414.043358:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320415.070211:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320415.968125:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985419648:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320415.968222:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320415.968344:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985419648 00000100:00100000:1.0:1767320415.968353:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985419648 00000100:00100000:1.0:1767320415.968416:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2586 00000100:00100000:1.0:1767320415.968425:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985419648:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320415.968451:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320415.968497:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985419648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (274us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320415.968504:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2586 00000100:00080000:3.0:1767320415.968571:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320415.968604:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985419648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320415.969671:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985419776:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320415.969722:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320415.969824:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985419776 00000100:00100000:1.0:1767320415.969833:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985419776 00000100:00100000:1.0:1767320415.969895:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2587 00000100:00100000:1.0:1767320415.969903:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985419776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320415.969927:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320415.969976:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985419776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320415.969983:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2587 00000100:00080000:2.0:1767320415.970031:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320415.970057:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985419776:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320416.091450:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320416.433570:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320416.433746:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920615936 00000100:00100000:1.0:1767320416.433755:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920615936 00000100:00100000:1.0:1767320416.433817:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5207 00000100:00100000:1.0:1767320416.433827:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920615936:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320416.433892:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920615936:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 67us (330us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320416.433900:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5207 00000100:00080000:2.0:1767320416.433970:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:1.0:1767320416.448154:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920615680 offset 0 length 520: 4 00000100:00100000:1.0:1767320416.448230:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320416.448435:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920615808 00000100:00100000:3.0:1767320416.448443:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920615808 00000100:00100000:3.0:1767320416.448488:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5881 00000100:00100000:3.0:1767320416.448494:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920615808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320416.448516:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320416.448581:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920615808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 86us (351us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320416.448588:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5881 00000100:00100000:2.0:1767320416.475306:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320391/real 1767320391] req@ffff8ce1b357a680 x1853163985413376/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320416 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320416.475338:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b357a680 x1853163985413376/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320416 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320416.475356:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320416.475382:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320416.475386:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985413376:0@lo:250:kworker.0 00000100:00080000:1.0:1767320416.475468:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320416.475474:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320416.475477:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320416.475482:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5640 00000100:00080000:1.0:1767320416.475486:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320416.475491:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320416.475523:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524e300 x1853163985419904/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320416.475551:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320416.475554:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320416.475556:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320416.475559:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5660 00000100:00080000:1.0:1767320416.475561:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320416.475564:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320416.475576:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985420032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320416.475584:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320416.475587:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320416.475589:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320416.475591:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5660 00000100:00080000:1.0:1767320416.475593:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320416.475596:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320416.475604:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c700 x1853163985420160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320416.475648:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320416.475652:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320416.475654:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320416.475657:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5660 00000100:00080000:1.0:1767320416.475659:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320416.475662:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320416.475672:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d180 x1853163985420416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320416.475685:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320416.475688:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320416.475690:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320416.475693:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5660 00000100:00080000:1.0:1767320416.475695:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320416.475697:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320416.475704:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985420672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320416.475761:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985419904:0@lo:250:kworker.0 00000400:00000100:2.0:1767320416.475802:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985419904 offset 0 length 520: 4 00000100:00100000:2.0:1767320416.475835:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985420032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320416.475858:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320416.475880:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985420160:0@lo:38:kworker.0 00000100:00100000:2.0:1767320416.475897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320416.475912:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985420416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320416.475926:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320416.475937:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985420672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320416.475950:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320416.475977:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420160 00000100:00100000:2.0:1767320416.475983:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420160 00000100:00100000:2.0:1767320416.476009:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1961 00000100:00100000:2.0:1767320416.476015:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985420160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320416.476032:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320416.476064:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985420160:12345-0@lo:38:kworker.0 Request processed in 48us (165us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320416.476081:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1961 00000100:00100000:2.0:1767320416.476101:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420032 00000100:00100000:2.0:1767320416.476104:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420032 00000100:00100000:2.0:1767320416.476118:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420416 00000100:00100000:2.0:1767320416.476120:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420416 00000100:00100000:2.0:1767320416.476129:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420672 00000100:00100000:2.0:1767320416.476131:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420672 00000100:00100000:2.0:1767320416.476141:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5882 00000100:00100000:2.0:1767320416.476145:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985420032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320416.476153:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320416.476193:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985420032:12345-0@lo:38:kworker.0 Request processed in 46us (333us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320416.476198:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5882 00000100:00100000:2.0:1767320416.476207:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5883 00000100:00100000:2.0:1767320416.476211:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985420416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320416.476219:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320416.476235:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985420416:12345-0@lo:38:kworker.0 Request processed in 23us (308us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320416.476240:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5883 00000100:00100000:2.0:1767320416.476247:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5884 00000100:00100000:2.0:1767320416.476251:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985420672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320416.476257:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320416.476271:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985420672:12345-0@lo:38:kworker.0 Request processed in 20us (321us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320416.476276:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5884 00000100:00080000:2.0:1767320416.476312:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320416.476316:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320416.476319:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985420032:0@lo:38:kworker.0 00000100:00080000:2.0:1767320416.476325:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320416.476328:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320416.476331:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985420160:0@lo:38:kworker.0 00000100:00080000:2.0:1767320416.476337:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320416.476340:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320416.476342:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985420416:0@lo:38:kworker.0 00000100:00080000:2.0:1767320416.476348:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320416.476351:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320416.476353:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985420672:0@lo:38:kworker.0 00000100:00100000:3.0:1767320416.476525:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985420288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320416.476553:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320416.476578:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320416.476582:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985420544:0@lo:400:kworker.0 00000100:00100000:3.0:1767320416.476599:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320416.476626:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420288 00000100:00100000:3.0:1767320416.476630:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420288 00000100:00100000:3.0:1767320416.476673:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420544 00000100:00100000:3.0:1767320416.476676:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420544 00000100:00100000:3.0:1767320416.476690:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5208 00000100:00100000:3.0:1767320416.476695:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985420288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320416.476728:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985420288:12345-0@lo:400:kworker.0 Request processed in 33us (174us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320416.476733:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5208 00000100:00100000:3.0:1767320416.476745:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5209 00000100:00100000:3.0:1767320416.476749:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985420544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320416.476767:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985420544:12345-0@lo:400:kworker.0 Request processed in 18us (167us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320416.476772:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5209 00000100:00080000:3.0:1767320416.476784:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320416.476802:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985420544:0@lo:400:kworker.0 00000100:00100000:3.0:1767320416.476809:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985420288:0@lo:400:kworker.0 00010000:00010000:0.0:1767320417.115657:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320418.139499:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320419.165824:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320420.187480:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320421.085033:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320421.085056:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985420800:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320421.085129:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320421.085244:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420800 00000100:00100000:1.0:1767320421.085252:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420800 00000100:00100000:1.0:1767320421.085309:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2588 00000100:00100000:1.0:1767320421.085319:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985420800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320421.085348:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320421.085390:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985420800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320421.085397:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2588 00000100:00080000:3.0:1767320421.085816:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320421.085854:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985420800:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320421.086196:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320421.086206:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985420928:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320421.086247:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320421.086339:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985420928 00000100:00100000:1.0:1767320421.086345:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985420928 00000100:00100000:1.0:1767320421.086395:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2589 00000100:00100000:1.0:1767320421.086403:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985420928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320421.086434:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320421.086469:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985420928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (220us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320421.086475:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2589 00000100:00080000:2.0:1767320421.088690:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320421.088816:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985420928:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320421.211388:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320421.545670:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320421.545786:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616064 00000100:00100000:2.0:1767320421.545803:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616064 00000100:00100000:2.0:1767320421.545870:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5885 00000100:00100000:2.0:1767320421.545879:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920616064:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320421.545903:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320421.545985:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920616064:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 106us (330us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320421.545992:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5885 00000100:00100000:0.0:1767320421.557413:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320421.557533:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616192 00000100:00100000:3.0:1767320421.557557:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616192 00000100:00100000:3.0:1767320421.557651:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5210 00000100:00100000:3.0:1767320421.557665:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920616192:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320421.557779:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920616192:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 117us (381us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320421.557788:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5210 00000100:00080000:2.0:1767320421.557918:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320421.595467:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.595482:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320421.595486:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.595491:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.595497:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.595508:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5664 00000100:00080000:1.0:1767320421.595515:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.595524:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320421.595591:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524fb80 x1853163985421056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320421.599604:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.599619:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.599624:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.599650:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5664 00000100:00080000:1.0:1767320421.599657:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.599667:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320421.599722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985421056:0@lo:38:kworker.0 00000100:00100000:1.0:1767320421.599733:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d500 x1853163985421184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320421.599820:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.599826:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.599828:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.599837:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5664 00000100:00080000:1.0:1767320421.599841:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.599845:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320421.599859:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320421.599862:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d880 x1853163985421440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320421.599881:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985421312:0@lo:400:kworker.0 00000100:00080000:1.0:1767320421.599889:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.599895:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.599898:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.599904:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5664 00000100:00100000:0.0:1767320421.599924:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320421.599966:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.599971:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320421.599987:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d180 x1853163985421696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320421.600073:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421056 00000100:00100000:1.0:1767320421.600096:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421056 00000100:00100000:1.0:1767320421.600151:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5886 00000100:00100000:1.0:1767320421.600161:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985421056:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320421.600213:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320421.600437:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320421.600452:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985421568:0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.604419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985421184:0@lo:38:kworker.0 00000100:00100000:2.0:1767320421.604431:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320421.604513:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421312 00000100:00100000:2.0:1767320421.604525:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421312 00000100:00100000:2.0:1767320421.604637:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5211 00000100:00100000:2.0:1767320421.604649:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985421312:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320421.604752:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985421312:12345-0@lo:400:kworker.0 Request processed in 103us (332us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320421.604762:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5211 00000100:00080000:2.0:1767320421.604791:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320421.604835:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985421312:0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.604933:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320421.604981:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985421440:0@lo:38:kworker.0 00000100:00100000:0.0:1767320421.605014:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320421.605019:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421184 00000100:00100000:2.0:1767320421.605028:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421184 00000100:00100000:0.0:1767320421.605035:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985421696:0@lo:38:kworker.0 00000100:00100000:2.0:1767320421.605078:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1962 00000100:00100000:0.0:1767320421.605085:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320421.605091:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985421184:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320421.605118:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985421056:12345-0@lo:38:kworker.0 Request processed in 4950us (5189us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767320421.605120:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320421.605137:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5886 00000100:00080000:0.0:1767320421.605145:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320421.605154:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320421.605160:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985421056:0@lo:38:kworker.0 00000100:00100000:1.0:1767320421.605202:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421440 00000100:00100000:1.0:1767320421.605209:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421440 00000100:00080000:0.0:1767320421.605218:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320421.605224:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320421.605226:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985421184:0@lo:38:kworker.0 00000100:00100000:1.0:1767320421.605250:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421696 00000100:00100000:1.0:1767320421.605257:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421696 00000100:00100000:1.0:1767320421.605287:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5887 00000100:00100000:1.0:1767320421.605294:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985421440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320421.605312:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320421.605366:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985421440:12345-0@lo:38:kworker.0 Request processed in 72us (352us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320421.605372:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5887 00000100:00100000:1.0:1767320421.605385:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5888 00000100:00100000:1.0:1767320421.605389:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985421696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320421.605399:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320421.605425:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985421696:12345-0@lo:38:kworker.0 Request processed in 33us (342us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320421.605430:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5888 00000100:00100000:2.0:1767320421.606402:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985421184:12345-0@lo:38:kworker.0 Request processed in 1304us (1463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320421.606415:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1962 00000100:00080000:0.0:1767320421.606670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320421.606676:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320421.606680:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985421440:0@lo:38:kworker.0 00000100:00080000:0.0:1767320421.606692:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320421.606695:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320421.606699:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985421696:0@lo:38:kworker.0 00000100:00100000:3.0:1767320421.609741:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320421.610096:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421568 00000100:00100000:2.0:1767320421.610113:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421568 00000100:00100000:2.0:1767320421.610265:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5212 00000100:00100000:2.0:1767320421.610275:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985421568:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320421.610365:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985421568:12345-0@lo:400:kworker.0 Request processed in 91us (635us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320421.610372:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5212 00000100:00100000:1.0:1767320421.610583:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985421568:0@lo:400:kworker.0 00000100:00080000:0.0:1767320421.610674:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320422.235530:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320423.259582:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320424.283425:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320425.309339:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320426.203709:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320426.203729:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985421824:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320426.203805:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.203932:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421824 00000100:00100000:2.0:1767320426.203942:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421824 00000100:00100000:2.0:1767320426.204021:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2590 00000100:00100000:2.0:1767320426.204031:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985421824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320426.204063:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320426.204118:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985421824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320426.204125:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2590 00000100:00100000:2.0:1767320426.204195:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985421824:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320426.204263:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320426.204439:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985421952:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320426.204477:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320426.204756:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985421952 00000100:00100000:1.0:1767320426.204762:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985421952 00000100:00100000:1.0:1767320426.204805:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2591 00000100:00100000:1.0:1767320426.204821:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985421952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320426.204839:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320426.204888:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985421952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (410us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320426.204894:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2591 00000100:00080000:3.0:1767320426.204940:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320426.205008:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985421952:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320426.331606:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320426.666562:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320426.666662:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616448 00000100:00100000:3.0:1767320426.666672:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616448 00000100:00100000:3.0:1767320426.666857:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5213 00000100:00080000:1.0:1767320426.666858:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320426.666875:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920616448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320426.666957:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920616448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 88us (403us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320426.666965:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5213 00000100:00100000:1.0:1767320426.667048:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320426.667197:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616320 00000100:00100000:3.0:1767320426.667202:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616320 00000100:00100000:3.0:1767320426.667232:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5889 00000100:00100000:3.0:1767320426.667476:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920616320:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320426.667498:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320426.667550:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920616320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 75us (502us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320426.681437:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5889 00000100:00080000:1.0:1767320426.715630:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.715637:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320426.715640:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.715644:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.715648:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.715656:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.715660:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320426.715667:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320426.715701:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524dc00 x1853163985422080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320426.715725:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.715729:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.715731:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.715734:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.715736:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320426.715740:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320426.715754:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f100 x1853163985422208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320426.715782:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.715785:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.715787:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.715791:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.715793:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320426.715796:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320426.715806:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985422464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320426.715820:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.715823:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.715824:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.715827:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.715828:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320426.715831:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320426.715842:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d880 x1853163985422720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320426.715868:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985422336:0@lo:400:kworker.0 00000100:00100000:1.0:1767320426.715932:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320426.715994:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320426.716000:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985422592:0@lo:400:kworker.0 00000100:00100000:1.0:1767320426.716033:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320426.716061:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422336 00000100:00100000:1.0:1767320426.716068:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422336 00000100:00100000:1.0:1767320426.716112:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422592 00000100:00100000:1.0:1767320426.716115:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422592 00000100:00100000:1.0:1767320426.716133:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5214 00000100:00100000:1.0:1767320426.716140:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985422336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320426.716208:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985422336:12345-0@lo:400:kworker.0 Request processed in 66us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320426.716215:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5214 00000100:00100000:1.0:1767320426.716230:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5215 00000100:00100000:1.0:1767320426.716237:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985422592:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320426.716261:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985422592:12345-0@lo:400:kworker.0 Request processed in 26us (241us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320426.716266:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5215 00000100:00080000:1.0:1767320426.716281:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320426.716311:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985422592:0@lo:400:kworker.0 00000100:00100000:1.0:1767320426.716320:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985422336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320426.717337:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985422080:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.717382:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.717423:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985422208:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.717446:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.717488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985422464:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.717506:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.717537:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985422720:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.717559:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.717595:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422080 00000100:00100000:2.0:1767320426.717601:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422080 00000100:00100000:2.0:1767320426.717640:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422464 00000100:00100000:2.0:1767320426.717643:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422464 00000100:00100000:2.0:1767320426.717652:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422720 00000100:00100000:2.0:1767320426.717654:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422720 00000100:00100000:2.0:1767320426.717664:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5890 00000100:00100000:2.0:1767320426.717670:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985422080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320426.717689:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320426.717725:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985422080:12345-0@lo:38:kworker.0 Request processed in 54us (342us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320426.717731:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5890 00000100:00100000:2.0:1767320426.717747:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5891 00000100:00100000:2.0:1767320426.717751:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985422464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320426.717759:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320426.717779:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985422464:12345-0@lo:38:kworker.0 Request processed in 27us (271us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320426.717785:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5891 00000100:00100000:2.0:1767320426.717794:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5892 00000100:00100000:2.0:1767320426.717798:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985422720:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320426.717807:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320426.717827:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985422720:12345-0@lo:38:kworker.0 Request processed in 27us (267us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320426.717832:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5892 00000100:00080000:2.0:1767320426.717869:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320426.717874:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320426.717877:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985422080:0@lo:38:kworker.0 00000100:00080000:2.0:1767320426.717884:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320426.717887:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320426.717889:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985422464:0@lo:38:kworker.0 00000100:00080000:2.0:1767320426.717895:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320426.717898:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320426.717901:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985422720:0@lo:38:kworker.0 00000100:00100000:0.0:1767320426.718002:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422208 00000100:00100000:0.0:1767320426.718006:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422208 00000100:00100000:0.0:1767320426.718036:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1963 00000100:00100000:0.0:1767320426.718040:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985422208:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320426.718050:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320426.718085:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985422208:12345-0@lo:38:kworker.0 Request processed in 43us (637us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320426.718091:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1963 00000100:00080000:0.0:1767320426.718113:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320426.718117:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320426.718119:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985422208:0@lo:38:kworker.0 00010000:00010000:1.0:1767320427.355485:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320428.379405:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320429.406897:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320430.429596:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320431.325083:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985422848:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320431.325155:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320431.325761:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320431.325772:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985422976:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320431.325800:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320431.325879:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422848 00000100:00100000:3.0:1767320431.325886:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422848 00000100:00100000:3.0:1767320431.325940:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985422976 00000100:00100000:3.0:1767320431.325943:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985422976 00000100:00100000:3.0:1767320431.325962:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2592 00000100:00100000:3.0:1767320431.325969:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985422848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320431.325996:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320431.326043:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985422848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (888us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320431.326051:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2592 00000100:00100000:3.0:1767320431.326066:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2593 00000100:00100000:3.0:1767320431.326071:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985422976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320431.326084:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320431.326104:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985422976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 33us (303us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320431.326110:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2593 00000100:00080000:3.0:1767320431.326224:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320431.326228:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320431.326572:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985422976:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320431.326589:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985422848:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320431.452225:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320431.787771:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320431.787932:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320431.788188:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616704 00000100:00100000:2.0:1767320431.788198:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616704 00000100:00080000:1.0:1767320431.788324:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320431.788427:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616576 00000100:00100000:2.0:1767320431.788432:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616576 00000100:00100000:3.0:1767320431.788492:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5216 00000100:00100000:3.0:1767320431.788503:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920616704:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320431.788567:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920616704:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 67us (806us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320431.788575:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5216 00000100:00100000:3.0:1767320431.788593:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5893 00000100:00100000:3.0:1767320431.788598:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920616576:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320431.788615:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320431.788633:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920616576:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 35us (703us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320431.788640:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5893 00000100:00080000:2.0:1767320431.838886:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320431.838898:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320431.838901:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320431.838905:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320431.838910:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320431.838918:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5674 00000100:00080000:2.0:1767320431.838923:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320431.838929:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320431.838970:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8bb80 x1853163985423104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320431.838990:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320431.838993:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320431.838996:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320431.838998:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5674 00000100:00080000:2.0:1767320431.839001:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320431.839004:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320431.839015:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a680 x1853163985423232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320431.839063:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320431.839067:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320431.839069:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320431.839072:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5674 00000100:00080000:2.0:1767320431.839075:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320431.839078:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320431.839087:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b800 x1853163985423488/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320431.839102:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320431.839105:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320431.839107:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320431.839110:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5674 00000100:00080000:2.0:1767320431.839112:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320431.839115:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320431.839123:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7100 x1853163985423744/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320431.839195:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985424000:0@lo:400:kworker.0 00000100:00100000:2.0:1767320431.839258:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320431.839329:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320431.839335:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985423872:0@lo:400:kworker.0 00000100:00100000:2.0:1767320431.839370:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320431.839508:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985423360:0@lo:400:kworker.0 00000100:00100000:1.0:1767320431.839539:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320431.839575:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320431.839580:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985423616:0@lo:400:kworker.0 00000100:00100000:1.0:1767320431.839601:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320431.840081:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424000 00000100:00100000:3.0:1767320431.840089:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424000 00000100:00100000:3.0:1767320431.840136:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985423872 00000100:00100000:3.0:1767320431.840139:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985423872 00000100:00100000:3.0:1767320431.840157:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3391 00000100:00100000:3.0:1767320431.840189:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985424000:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320431.840244:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985424000:12345-0@lo:400:kworker.0 Request processed in 79us (985us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320431.840252:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3391 00000100:00100000:3.0:1767320431.840267:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3392 00000100:00100000:3.0:1767320431.840272:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985423872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320431.840310:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985423872:12345-0@lo:400:kworker.0 Request processed in 37us (950us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320431.840315:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3392 00000100:00100000:3.0:1767320431.840350:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985423872:0@lo:400:kworker.0 00000100:00100000:3.0:1767320431.840372:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985424000:0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.840593:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985423104:0@lo:38:kworker.0 00000100:00100000:0.0:1767320431.840626:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320431.840663:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985423232:0@lo:38:kworker.0 00000100:00100000:0.0:1767320431.840686:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320431.840714:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985423488:0@lo:38:kworker.0 00000100:00100000:0.0:1767320431.840733:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320431.840750:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985423744:0@lo:38:kworker.0 00000100:00100000:0.0:1767320431.840770:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320431.840804:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985423360 00000100:00100000:0.0:1767320431.840808:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985423360 00000100:00100000:0.0:1767320431.840840:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985423616 00000100:00100000:0.0:1767320431.840842:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985423616 00000100:00100000:0.0:1767320431.840851:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985423104 00000100:00100000:0.0:1767320431.840853:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985423104 00000100:00100000:0.0:1767320431.840859:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985423488 00000100:00100000:0.0:1767320431.840861:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985423488 00000100:00100000:0.0:1767320431.840866:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985423744 00000100:00100000:0.0:1767320431.840868:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985423744 00000100:00100000:0.0:1767320431.840876:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5217 00000100:00100000:0.0:1767320431.840881:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985423360:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.840930:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985423360:12345-0@lo:400:kworker.0 Request processed in 48us (1389us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320431.840936:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5217 00000100:00100000:0.0:1767320431.840947:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5218 00000100:00100000:0.0:1767320431.840953:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985423616:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.840975:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985423616:12345-0@lo:400:kworker.0 Request processed in 22us (1373us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320431.840980:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5218 00000100:00100000:0.0:1767320431.840989:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5894 00000100:00100000:0.0:1767320431.840993:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985423104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320431.841009:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320431.841035:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985423104:12345-0@lo:38:kworker.0 Request processed in 41us (409us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320431.841041:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5894 00000100:00100000:0.0:1767320431.841051:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5895 00000100:00100000:0.0:1767320431.841054:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985423488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320431.841062:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320431.841082:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985423488:12345-0@lo:38:kworker.0 Request processed in 26us (347us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320431.841086:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5895 00000100:00100000:0.0:1767320431.841095:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5896 00000100:00100000:0.0:1767320431.841098:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985423744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320431.841105:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320431.841123:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985423744:12345-0@lo:38:kworker.0 Request processed in 24us (353us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320431.841129:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5896 00000100:00080000:0.0:1767320431.841141:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320431.841185:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985423616:0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.841195:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985423360:0@lo:400:kworker.0 00000100:00080000:0.0:1767320431.841216:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320431.841220:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320431.841223:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985423104:0@lo:38:kworker.0 00000100:00080000:0.0:1767320431.841229:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320431.841232:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320431.841234:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985423488:0@lo:38:kworker.0 00000100:00080000:0.0:1767320431.841241:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320431.841243:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320431.841246:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985423744:0@lo:38:kworker.0 00000100:00100000:3.0:1767320431.841631:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985423232 00000100:00100000:3.0:1767320431.841637:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985423232 00000100:00100000:3.0:1767320431.841676:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1964 00000100:00100000:3.0:1767320431.841681:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985423232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320431.841693:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320431.841733:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985423232:12345-0@lo:38:kworker.0 Request processed in 51us (1046us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320431.841739:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1964 00000100:00080000:1.0:1767320431.841797:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320431.841802:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320431.841805:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985423232:0@lo:38:kworker.0 00010000:00010000:1.0:1767320432.476193:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320433.499654:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320434.523488:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320435.547409:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320436.444358:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985424128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320436.444436:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320436.444635:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424128 00000100:00100000:2.0:1767320436.444644:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424128 00000100:00100000:2.0:1767320436.444713:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2594 00000100:00100000:2.0:1767320436.444723:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985424128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320436.444753:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320436.444796:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985424128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (359us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320436.444803:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2594 00000100:00080000:1.0:1767320436.445245:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320436.445296:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985424128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320436.445665:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320436.445677:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985424256:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320436.445715:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320436.445795:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424256 00000100:00100000:3.0:1767320436.445801:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424256 00000100:00100000:3.0:1767320436.445847:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2595 00000100:00100000:3.0:1767320436.445855:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985424256:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320436.446002:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:3.0:1767320436.446040:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320436.446096:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985424256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 242us (380us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320436.446104:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2595 00000100:00100000:1.0:1767320436.446231:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985424256:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320436.572874:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320436.905779:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320436.905825:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320436.906054:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616960 00000100:00100000:2.0:1767320436.906064:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616960 00000100:00100000:2.0:1767320436.906129:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920616832 00000100:00100000:2.0:1767320436.906133:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920616832 00000100:00100000:2.0:1767320436.906155:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5219 00000100:00100000:2.0:1767320436.906201:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920616960:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320436.906289:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920616960:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 126us (519us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320436.906297:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5219 00000100:00100000:2.0:1767320436.906316:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5897 00000100:00100000:2.0:1767320436.906320:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920616832:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320436.906341:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320436.906379:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920616832:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (553us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320436.906385:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5897 00000100:00080000:0.0:1767320436.906738:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320436.955330:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320436.955340:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320436.955343:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320436.955347:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320436.955351:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320436.955359:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5680 00000100:00080000:2.0:1767320436.955365:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320436.955372:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320436.955410:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985424384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320436.955434:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320436.955438:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320436.955440:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320436.955443:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5680 00000100:00080000:2.0:1767320436.955447:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320436.955450:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320436.955463:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2680 x1853163985424512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320436.955507:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320436.955511:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320436.955513:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320436.955517:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5680 00000100:00080000:2.0:1767320436.955520:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320436.955523:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320436.955537:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2a00 x1853163985424768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320436.955553:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320436.955556:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320436.955557:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320436.955561:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5680 00000100:00080000:2.0:1767320436.955563:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320436.955566:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320436.955575:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0e00 x1853163985425024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320436.955696:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985424640:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.955760:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320436.955831:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320436.955837:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985424896:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.955859:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320436.955890:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424640 00000100:00100000:0.0:1767320436.955897:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424640 00000100:00100000:0.0:1767320436.955938:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424896 00000100:00100000:0.0:1767320436.955940:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424896 00000100:00100000:0.0:1767320436.955959:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5220 00000100:00100000:0.0:1767320436.955966:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985424640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.956009:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985424640:12345-0@lo:400:kworker.0 Request processed in 43us (249us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320436.956016:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5220 00000100:00100000:0.0:1767320436.956030:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5221 00000100:00100000:0.0:1767320436.956035:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985424896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.956054:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985424896:12345-0@lo:400:kworker.0 Request processed in 19us (195us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320436.956059:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5221 00000100:00080000:0.0:1767320436.956075:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320436.956084:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985425280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.956107:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320436.956124:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320436.956129:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985425152:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.956148:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320436.956248:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985425280 00000100:00100000:3.0:1767320436.956262:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985425280 00000100:00100000:1.0:1767320436.956295:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985424384:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.956328:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320436.956338:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985425152 00000100:00100000:3.0:1767320436.956343:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985425152 00000100:00100000:1.0:1767320436.956367:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985424512:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.956389:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320436.956408:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985424768:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.956423:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320436.956434:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985425024:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.956449:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320436.956491:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3393 00000100:00100000:1.0:1767320436.956499:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985425280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320436.956560:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424384 00000100:00100000:2.0:1767320436.956564:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424384 00000100:00100000:2.0:1767320436.956584:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424768 00000100:00100000:2.0:1767320436.956586:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424768 00000100:00100000:2.0:1767320436.956592:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985425024 00000100:00100000:2.0:1767320436.956594:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985425024 00000100:00100000:2.0:1767320436.956603:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5898 00000100:00100000:2.0:1767320436.956608:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985424384:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320436.956624:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320436.956652:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985424384:12345-0@lo:38:kworker.0 Request processed in 44us (326us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320436.956659:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5898 00000100:00100000:2.0:1767320436.956671:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5899 00000100:00100000:2.0:1767320436.956675:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985424768:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320436.956683:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320436.956700:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985424768:12345-0@lo:38:kworker.0 Request processed in 25us (277us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320436.956706:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5899 00000100:00100000:2.0:1767320436.956727:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5900 00000100:00100000:2.0:1767320436.956731:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985425024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320436.956739:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320436.956757:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985425024:12345-0@lo:38:kworker.0 Request processed in 24us (307us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320436.956762:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5900 00000100:00100000:2.0:1767320436.956807:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985424512 00000100:00100000:2.0:1767320436.956810:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985424512 00000100:00100000:2.0:1767320436.956825:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1965 00000100:00100000:2.0:1767320436.956829:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985424512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320436.956838:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320436.956857:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985424512:12345-0@lo:38:kworker.0 Request processed in 27us (467us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320436.956865:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1965 00000100:00100000:3.0:1767320436.957332:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3394 00000100:00100000:3.0:1767320436.957349:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985425152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.960248:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985424896:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.960277:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985424640:0@lo:400:kworker.0 00000100:00080000:0.0:1767320436.960333:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320436.960339:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320436.960342:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985424384:0@lo:38:kworker.0 00000100:00080000:0.0:1767320436.960351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320436.960353:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320436.960356:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985424512:0@lo:38:kworker.0 00000100:00080000:0.0:1767320436.960367:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320436.960370:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320436.960373:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985424768:0@lo:38:kworker.0 00000100:00080000:0.0:1767320436.960381:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320436.960384:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320436.960386:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985425024:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.961910:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985425280:12345-0@lo:400:kworker.0 Request processed in 5404us (5793us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320436.961924:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3393 00000100:00100000:3.0:1767320436.963352:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985425152:12345-0@lo:400:kworker.0 Request processed in 5995us (7187us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320436.963381:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3394 00000100:00100000:0.0:1767320436.964250:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985425152:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.964282:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985425280:0@lo:400:kworker.0 00010000:00010000:1.0:1767320437.595481:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320438.619539:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320439.644882:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320440.670514:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320441.051714:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320416/real 1767320416] req@ffff8ce1b524e300 x1853163985419904/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320441 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320441.051746:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b524e300 x1853163985419904/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320441 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320441.051763:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320441.051789:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320441.051793:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985419904:0@lo:250:kworker.0 00000100:00080000:3.0:1767320441.051892:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320441.051897:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320441.051900:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320441.051904:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5664 00000100:00080000:3.0:1767320441.051908:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320441.051915:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320441.051941:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8000 x1853163985425408/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320441.051964:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320441.051967:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320441.051969:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320441.051972:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5685 00000100:00080000:3.0:1767320441.051974:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320441.051977:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320441.052068:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb800 x1853163985425536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320441.052078:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320441.052081:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320441.052083:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320441.052088:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5685 00000100:00080000:3.0:1767320441.052090:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320441.052095:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320441.052110:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9c00 x1853163985425664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320441.052141:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320441.052144:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320441.052146:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320441.052149:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5685 00000100:00080000:3.0:1767320441.052151:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320441.052155:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320441.052209:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beaa00 x1853163985425920/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320441.052229:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320441.052237:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985425792:0@lo:400:kworker.0 00000100:00080000:3.0:1767320441.052237:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320441.052240:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320441.052251:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5685 00000100:00080000:3.0:1767320441.052255:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320441.052259:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320441.052277:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bea300 x1853163985426176/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320441.052355:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320441.052442:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320441.052454:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985426048:0@lo:400:kworker.0 00000100:00100000:3.0:1767320441.052461:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985426304:0@lo:400:kworker.0 00000100:00100000:3.0:1767320441.052523:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320441.052595:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320441.052601:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985426432:0@lo:400:kworker.0 00000100:00100000:3.0:1767320441.052624:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320441.052719:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426304 00000100:00100000:3.0:1767320441.052727:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426304 00000100:00100000:1.0:1767320441.052738:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320441.052756:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320441.052793:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426432 00000100:00100000:1.0:1767320441.052803:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426432 00000100:00100000:3.0:1767320441.052808:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3395 00000100:00100000:3.0:1767320441.052822:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985426304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320441.052875:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3396 00000100:00100000:3.0:1767320441.052887:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985426304:12345-0@lo:400:kworker.0 Request processed in 67us (361us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320441.052889:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985426432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320441.052908:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3395 00000100:00100000:3.0:1767320441.052954:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985426304:0@lo:400:kworker.0 00000100:00100000:1.0:1767320441.052966:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985426432:12345-0@lo:400:kworker.0 Request processed in 77us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320441.052976:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3396 00000100:00100000:3.0:1767320441.052983:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985426432:0@lo:400:kworker.0 00000100:00080000:1.0:1767320441.053021:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320441.053232:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985425792 00000100:00100000:2.0:1767320441.053238:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985425792 00000100:00100000:2.0:1767320441.053286:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426048 00000100:00100000:2.0:1767320441.053290:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426048 00000100:00080000:1.0:1767320441.053307:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320441.053323:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5222 00000100:00100000:2.0:1767320441.053333:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985425792:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320441.053434:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5223 00000100:00100000:1.0:1767320441.053447:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985426048:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320441.056482:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985425792:12345-0@lo:400:kworker.0 Request processed in 3139us (4117us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320441.056497:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5222 00000100:00100000:2.0:1767320441.056559:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985425408:0@lo:250:kworker.0 00000400:00000100:2.0:1767320441.056602:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985425408 offset 0 length 520: 4 00000100:00100000:2.0:1767320441.056634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985425536:0@lo:38:kworker.0 00000100:00100000:2.0:1767320441.056661:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320441.056682:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985425664:0@lo:38:kworker.0 00000100:00100000:2.0:1767320441.056705:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320441.056722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985425920:0@lo:38:kworker.0 00000100:00100000:2.0:1767320441.056746:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320441.056760:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985426176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320441.056779:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320441.056816:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985425664 00000100:00100000:2.0:1767320441.056823:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985425664 00000100:00100000:2.0:1767320441.056852:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1966 00000100:00100000:2.0:1767320441.056859:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985425664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320441.056878:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320441.056929:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985425664:12345-0@lo:38:kworker.0 Request processed in 69us (222us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320441.056935:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1966 00000100:00100000:2.0:1767320441.056954:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985425536 00000100:00100000:2.0:1767320441.056957:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985425536 00000100:00100000:2.0:1767320441.056971:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985425920 00000100:00100000:2.0:1767320441.056973:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985425920 00000100:00100000:2.0:1767320441.056982:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426176 00000100:00100000:2.0:1767320441.056984:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426176 00000100:00100000:2.0:1767320441.056993:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5901 00000100:00100000:2.0:1767320441.056997:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985425536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320441.057006:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320441.057028:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985425536:12345-0@lo:38:kworker.0 Request processed in 30us (367us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320441.057034:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5901 00000100:00100000:2.0:1767320441.057043:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5902 00000100:00100000:2.0:1767320441.057047:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985425920:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320441.057055:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320441.057073:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985425920:12345-0@lo:38:kworker.0 Request processed in 25us (325us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320441.057078:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5902 00000100:00100000:2.0:1767320441.057087:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5903 00000100:00100000:2.0:1767320441.057091:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985426176:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320441.057099:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320441.057116:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985426176:12345-0@lo:38:kworker.0 Request processed in 24us (335us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320441.057121:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5903 00000100:00080000:2.0:1767320441.057160:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320441.057384:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320441.057389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985425536:0@lo:38:kworker.0 00000100:00080000:2.0:1767320441.057403:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320441.057406:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320441.057409:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985425664:0@lo:38:kworker.0 00000100:00080000:2.0:1767320441.057415:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320441.057419:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320441.057422:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985425920:0@lo:38:kworker.0 00000100:00080000:2.0:1767320441.057428:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320441.057431:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320441.057433:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985426176:0@lo:38:kworker.0 00000100:00100000:1.0:1767320441.059014:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985426048:12345-0@lo:400:kworker.0 Request processed in 5557us (6265us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320441.059030:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5223 00000100:00100000:2.0:1767320441.059390:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985426048:0@lo:400:kworker.0 00000100:00100000:2.0:1767320441.059405:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985425792:0@lo:400:kworker.0 00000100:00100000:0.0:1767320441.566018:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320441.566043:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985426560:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320441.566108:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320441.566260:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426560 00000100:00100000:2.0:1767320441.566277:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426560 00000100:00100000:2.0:1767320441.566333:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2596 00000100:00100000:2.0:1767320441.566342:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985426560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320441.566371:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320441.566611:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985426560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 270us (501us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320441.566619:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2596 00000100:00100000:2.0:1767320441.571376:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320441.571411:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985426688:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320441.571489:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320441.571591:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426688 00000100:00100000:0.0:1767320441.571602:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426688 00000100:00100000:0.0:1767320441.571658:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2597 00000100:00100000:0.0:1767320441.571667:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985426688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320441.571701:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320441.571743:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985426688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (253us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320441.571751:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2597 00000100:00100000:2.0:1767320441.571836:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985426688:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320441.576922:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985426560:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320441.692639:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767320442.026355:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920617088 offset 0 length 520: 4 00000100:00100000:1.0:1767320442.026406:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320442.026447:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320442.026523:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920617344 00000100:00100000:2.0:1767320442.026533:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920617344 00000100:00100000:2.0:1767320442.026604:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920617216 00000100:00100000:2.0:1767320442.026608:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920617216 00000100:00100000:2.0:1767320442.026627:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5224 00000100:00100000:2.0:1767320442.026637:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920617344:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320442.026708:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920617344:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 72us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320442.026716:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5224 00000100:00100000:2.0:1767320442.026733:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5904 00000100:00100000:2.0:1767320442.026738:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920617216:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320442.026757:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320442.026780:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920617216:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (332us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320442.026787:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5904 00000100:00080000:1.0:1767320442.027154:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320442.715863:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320443.740053:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320444.765869:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320445.788412:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320446.171528:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320446.171539:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320446.171542:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320446.171546:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320446.171552:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320446.171560:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5689 00000100:00080000:0.0:1767320446.171565:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320446.171571:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320446.171609:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4e00 x1853163985426816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320446.171636:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320446.171639:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320446.171641:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320446.171645:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5689 00000100:00080000:0.0:1767320446.171647:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320446.171650:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320446.171663:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4380 x1853163985426944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320446.171705:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320446.171709:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320446.171711:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320446.171714:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5689 00000100:00080000:0.0:1767320446.171717:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320446.171720:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320446.171734:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8bb80 x1853163985427200/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320446.171750:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320446.171753:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320446.171756:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320446.171759:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5689 00000100:00080000:0.0:1767320446.171761:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320446.171764:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320446.171775:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b800 x1853163985427456/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320446.171821:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985427584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320446.171892:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320446.172048:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985427328:0@lo:400:kworker.0 00000100:00100000:1.0:1767320446.172083:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320446.172090:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985426816:0@lo:38:kworker.0 00000100:00100000:1.0:1767320446.172138:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427328 00000100:00100000:2.0:1767320446.172141:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320446.172148:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427328 00000100:00100000:2.0:1767320446.172212:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427584 00000100:00100000:2.0:1767320446.172216:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427584 00000100:00100000:1.0:1767320446.172225:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426816 00000100:00100000:1.0:1767320446.172231:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426816 00000100:00100000:2.0:1767320446.172252:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3397 00000100:00100000:1.0:1767320446.172258:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5225 00000100:00100000:2.0:1767320446.172266:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985427584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320446.172272:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985427328:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320446.172322:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985427584:12345-0@lo:400:kworker.0 Request processed in 60us (429us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320446.172331:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985427328:12345-0@lo:400:kworker.0 Request processed in 63us (246us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320446.172332:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3397 00000100:00100000:1.0:1767320446.172343:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5225 00000100:00100000:2.0:1767320446.172365:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5905 00000100:00100000:2.0:1767320446.172370:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985426816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320446.172388:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320446.172425:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985427072:0@lo:400:kworker.0 00000100:00100000:2.0:1767320446.172437:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985426816:12345-0@lo:38:kworker.0 Request processed in 64us (296us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320446.172504:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985427584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320446.172524:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320446.172530:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985427712:0@lo:400:kworker.0 00000100:00100000:0.0:1767320446.172551:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320446.172583:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427712 00000100:00100000:0.0:1767320446.172587:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427712 00000100:00100000:0.0:1767320446.172613:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3398 00000100:00100000:0.0:1767320446.172620:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985427712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320446.172652:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985427712:12345-0@lo:400:kworker.0 Request processed in 31us (99us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320446.172658:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3398 00000100:00100000:0.0:1767320446.172682:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985427712:0@lo:400:kworker.0 00000100:00100000:0.0:1767320446.172719:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985427328:0@lo:400:kworker.0 00000100:00080000:0.0:1767320446.172752:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320446.172756:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320446.172758:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320446.172774:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320446.172780:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320446.172783:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985426816:0@lo:38:kworker.0 00000100:00100000:0.0:1767320446.172789:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985426944:0@lo:38:kworker.0 00000100:00100000:0.0:1767320446.172812:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320446.172835:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985427200:0@lo:38:kworker.0 00000100:00100000:0.0:1767320446.172854:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320446.172868:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985427456:0@lo:38:kworker.0 00000100:00100000:0.0:1767320446.172887:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320446.172919:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427200 00000100:00100000:0.0:1767320446.172923:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427200 00000100:00100000:3.0:1767320446.173255:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320446.173370:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985426944 00000100:00100000:3.0:1767320446.173375:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985426944 00000100:00100000:3.0:1767320446.173413:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1967 00000100:00100000:3.0:1767320446.173418:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985426944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320446.173432:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320446.173467:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985426944:12345-0@lo:38:kworker.0 Request processed in 47us (653us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320446.173474:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1967 00000100:00080000:3.0:1767320446.173502:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320446.173506:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320446.173509:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985426944:0@lo:38:kworker.0 00000100:00100000:3.0:1767320446.173543:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427456 00000100:00100000:3.0:1767320446.173547:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427456 00000100:00100000:1.0:1767320446.182344:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427072 00000100:00100000:1.0:1767320446.182358:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427072 00000100:00100000:1.0:1767320446.182426:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5226 00000100:00100000:1.0:1767320446.182436:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985427072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320446.182966:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985427072:12345-0@lo:400:kworker.0 Request processed in 530us (9709us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320446.182975:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5226 00000100:00100000:1.0:1767320446.183037:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985427072:0@lo:400:kworker.0 00000100:00080000:1.0:1767320446.183063:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320446.183802:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5905 00000100:00100000:0.0:1767320446.184583:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5906 00000100:00100000:0.0:1767320446.184593:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985427200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320446.184617:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320446.184698:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985427200:12345-0@lo:38:kworker.0 Request processed in 104us (11841us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320446.184704:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5906 00000100:00080000:1.0:1767320446.184846:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320446.184854:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320446.184858:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985427200:0@lo:38:kworker.0 00000100:00100000:3.0:1767320446.185585:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5907 00000100:00100000:3.0:1767320446.185593:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985427456:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320446.185608:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320446.185663:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985427456:12345-0@lo:38:kworker.0 Request processed in 69us (12774us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320446.185669:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5907 00000100:00080000:1.0:1767320446.186123:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320446.186128:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320446.186132:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985427456:0@lo:38:kworker.0 00000100:00100000:1.0:1767320446.683462:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985427840:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320446.683538:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320446.683719:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985427968:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320446.683749:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320446.683789:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427840 00000100:00100000:0.0:1767320446.683797:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427840 00000100:00100000:0.0:1767320446.683839:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985427968 00000100:00100000:0.0:1767320446.683842:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985427968 00000100:00100000:0.0:1767320446.683861:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2598 00000100:00100000:0.0:1767320446.683869:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985427840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320446.683911:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320446.683982:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2599 00000100:00100000:2.0:1767320446.683992:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985427968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320446.684008:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320446.684039:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985427968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (288us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320446.684046:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2599 00000100:00100000:2.0:1767320446.685565:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985427968:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320446.685772:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985427840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1901us (2232us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320446.685782:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2598 00000100:00100000:1.0:1767320446.692699:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985427840:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320446.811548:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320447.145846:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320447.153818:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920617472 00000100:00100000:0.0:1767320447.153839:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920617472 00000100:00100000:0.0:1767320447.153905:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5908 00000100:00100000:0.0:1767320447.153923:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920617472:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320447.153947:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320447.154030:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920617472:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 107us (8194us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320447.154038:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5908 00000100:00100000:2.0:1767320447.154788:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320447.154900:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920617600 00000100:00100000:3.0:1767320447.154907:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920617600 00000100:00100000:3.0:1767320447.155116:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5227 00000100:00100000:3.0:1767320447.155126:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920617600:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320447.155327:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920617600:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 201us (542us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320447.155336:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5227 00000100:00080000:1.0:1767320447.155708:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320447.835366:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320448.859717:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320449.883410:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320450.908871:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320451.291341:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320451.291352:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320451.291355:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320451.291359:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320451.291364:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320451.291372:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5694 00000100:00080000:0.0:1767320451.291377:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320451.291383:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320451.291423:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8700 x1853163985428096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320451.291469:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320451.291472:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320451.291474:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320451.291477:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5694 00000100:00080000:0.0:1767320451.291481:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320451.291485:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320451.291499:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8a80 x1853163985428224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320451.291525:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320451.291528:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320451.291530:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320451.291533:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5694 00000100:00080000:0.0:1767320451.291536:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320451.291539:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320451.291550:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9180 x1853163985428480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320451.291567:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320451.291570:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320451.291572:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320451.291575:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5694 00000100:00080000:0.0:1767320451.291577:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320451.291580:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320451.291589:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdad80 x1853163985428736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320451.291647:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985428864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320451.291712:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320451.291719:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985428096:0@lo:38:kworker.0 00000100:00100000:1.0:1767320451.291747:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320451.291764:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320451.291770:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985428992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320451.291792:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320451.291860:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985428224:0@lo:38:kworker.0 00000100:00100000:1.0:1767320451.291897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320451.291917:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985428480:0@lo:38:kworker.0 00000100:00100000:1.0:1767320451.291936:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320451.291949:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985428736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320451.291966:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320451.291968:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985428352:0@lo:400:kworker.0 00000100:00100000:2.0:1767320451.292130:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428224 00000100:00100000:2.0:1767320451.292137:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428224 00000100:00100000:2.0:1767320451.292188:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1968 00000100:00100000:2.0:1767320451.292195:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985428224:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320451.292212:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320451.292248:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985428224:12345-0@lo:38:kworker.0 Request processed in 53us (351us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320451.292254:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1968 00000100:00100000:3.0:1767320451.295549:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428864 00000100:00100000:3.0:1767320451.295562:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428864 00000100:00100000:3.0:1767320451.295626:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428992 00000100:00100000:3.0:1767320451.295628:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428992 00000100:00100000:3.0:1767320451.295648:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3399 00000100:00100000:3.0:1767320451.295656:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985428864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320451.295726:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985428864:12345-0@lo:400:kworker.0 Request processed in 71us (4014us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320451.295734:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3399 00000100:00100000:3.0:1767320451.295749:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3400 00000100:00100000:3.0:1767320451.295765:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985428992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320451.295788:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985428992:12345-0@lo:400:kworker.0 Request processed in 35us (3995us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320451.295794:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3400 00000100:00080000:3.0:1767320451.295810:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320451.295813:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320451.295844:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985428992:0@lo:400:kworker.0 00000100:00100000:3.0:1767320451.295855:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985428864:0@lo:400:kworker.0 00000100:00100000:3.0:1767320451.295912:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985428608:0@lo:400:kworker.0 00000100:00100000:3.0:1767320451.299799:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320451.302549:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320451.302564:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320451.302570:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985428224:0@lo:38:kworker.0 00000100:00100000:3.0:1767320451.305534:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320451.310067:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428096 00000100:00100000:2.0:1767320451.310086:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428096 00000100:00100000:2.0:1767320451.310159:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428480 00000100:00100000:2.0:1767320451.310173:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428480 00000100:00100000:2.0:1767320451.310190:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428736 00000100:00100000:2.0:1767320451.310193:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428736 00000100:00100000:2.0:1767320451.310201:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428608 00000100:00100000:2.0:1767320451.310204:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428608 00000100:00100000:2.0:1767320451.310242:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5228 00000100:00100000:2.0:1767320451.310254:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985428608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320451.310342:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985428608:12345-0@lo:400:kworker.0 Request processed in 92us (10549us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320451.310349:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5228 00000100:00100000:2.0:1767320451.310367:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5909 00000100:00100000:2.0:1767320451.310371:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985428096:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320451.310390:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320451.310426:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985428096:12345-0@lo:38:kworker.0 Request processed in 53us (18679us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320451.312670:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5909 00000100:00100000:2.0:1767320451.312717:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5910 00000100:00100000:2.0:1767320451.312727:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985428480:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320451.312751:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320451.313699:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985428480:12345-0@lo:38:kworker.0 Request processed in 968us (21758us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320451.313705:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5910 00000100:00100000:2.0:1767320451.313721:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5911 00000100:00100000:2.0:1767320451.313724:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985428736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320451.313736:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320451.313753:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985428736:12345-0@lo:38:kworker.0 Request processed in 27us (21786us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320451.314623:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5911 00000100:00080000:2.0:1767320451.314674:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320451.314751:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985428608:0@lo:400:kworker.0 00000100:00080000:0.0:1767320451.314921:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320451.314927:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320451.314930:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985428096:0@lo:38:kworker.0 00000100:00080000:0.0:1767320451.314940:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320451.314943:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320451.314945:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985428480:0@lo:38:kworker.0 00000100:00080000:0.0:1767320451.314953:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320451.314956:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320451.314958:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985428736:0@lo:38:kworker.0 00000100:00100000:0.0:1767320451.315139:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985428352 00000100:00100000:0.0:1767320451.315524:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985428352 00000100:00100000:0.0:1767320451.315585:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5229 00000100:00100000:0.0:1767320451.315594:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985428352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320451.315656:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985428352:12345-0@lo:400:kworker.0 Request processed in 63us (10130us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320451.315664:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5229 00000100:00080000:3.0:1767320451.315734:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320451.315759:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985428352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320451.803593:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985429120:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320451.803628:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985429248:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320451.806609:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320451.806745:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985429120 00000100:00100000:2.0:1767320451.806755:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985429120 00000100:00100000:2.0:1767320451.806813:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2600 00000100:00100000:2.0:1767320451.806824:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985429120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320451.806857:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320451.806957:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320451.807020:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985429248 00000100:00100000:1.0:1767320451.807024:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985429248 00000100:00100000:1.0:1767320451.807048:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2601 00000100:00100000:1.0:1767320451.807054:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985429248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320451.807072:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320451.807102:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985429248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (146us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320451.807110:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2601 00000100:00100000:1.0:1767320451.807176:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985429248:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320451.811323:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985429120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4489us (4709us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320451.811338:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2600 00000100:00100000:0.0:1767320451.812842:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985429120:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320451.932400:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320452.273546:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320452.273619:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920617856 00000100:00100000:1.0:1767320452.273636:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920617856 00000100:00100000:1.0:1767320452.273815:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5230 00000100:00100000:1.0:1767320452.273825:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920617856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320452.273931:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920617856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 107us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320452.273939:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5230 00000100:00080000:3.0:1767320452.274546:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320452.282430:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320452.282561:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920617728 00000100:00100000:0.0:1767320452.282572:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920617728 00000100:00100000:0.0:1767320452.282625:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5912 00000100:00100000:0.0:1767320452.282633:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920617728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320452.282659:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320452.282744:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920617728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 110us (326us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320452.282752:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5912 00010000:00010000:1.0:1767320452.957473:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320453.980009:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320455.004019:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320456.027472:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320456.411492:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320456.411503:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320456.411506:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320456.411510:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320456.411514:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320456.411522:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5699 00000100:00080000:2.0:1767320456.411527:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320456.411534:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320456.411575:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985429376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320456.411601:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320456.411605:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320456.411607:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320456.411611:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5699 00000100:00080000:2.0:1767320456.411613:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320456.411617:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320456.411629:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1880 x1853163985429504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320456.411676:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320456.411680:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320456.411681:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320456.411686:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5699 00000100:00080000:2.0:1767320456.411688:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320456.411693:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320456.411705:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3100 x1853163985429760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320456.411721:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320456.411725:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320456.411727:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320456.411730:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5699 00000100:00080000:2.0:1767320456.411732:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320456.411735:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320456.411745:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2a00 x1853163985430016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320456.411929:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985429888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320456.412011:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320456.412087:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320456.412093:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985429632:0@lo:400:kworker.0 00000100:00100000:1.0:1767320456.412117:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320456.412142:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985430272:0@lo:400:kworker.0 00000100:00100000:1.0:1767320456.412208:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320456.412228:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320456.412233:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985430144:0@lo:400:kworker.0 00000100:00100000:1.0:1767320456.412255:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320456.412496:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985429888 00000100:00100000:2.0:1767320456.412504:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985429888 00000100:00100000:2.0:1767320456.412560:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985429632 00000100:00100000:2.0:1767320456.412563:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985429632 00000100:00100000:2.0:1767320456.412586:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5231 00000100:00100000:2.0:1767320456.412594:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985429888:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320456.412649:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985429888:12345-0@lo:400:kworker.0 Request processed in 56us (638us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320456.412656:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5231 00000100:00100000:2.0:1767320456.412673:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5232 00000100:00100000:2.0:1767320456.412679:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985429632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320456.412707:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985429632:12345-0@lo:400:kworker.0 Request processed in 28us (590us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320456.412713:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5232 00000100:00100000:2.0:1767320456.412753:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985429632:0@lo:400:kworker.0 00000100:00100000:2.0:1767320456.412762:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985429888:0@lo:400:kworker.0 00000100:00100000:0.0:1767320456.412930:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985429376:0@lo:38:kworker.0 00000100:00100000:0.0:1767320456.412975:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320456.413008:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985429504:0@lo:38:kworker.0 00000100:00100000:0.0:1767320456.413032:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320456.413053:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985429760:0@lo:38:kworker.0 00000100:00100000:0.0:1767320456.413072:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320456.413085:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985430016:0@lo:38:kworker.0 00000100:00100000:0.0:1767320456.413101:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320456.413128:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985429504 00000100:00100000:0.0:1767320456.413132:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985429504 00000100:00100000:0.0:1767320456.413153:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1969 00000100:00100000:0.0:1767320456.413157:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985429504:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320456.413395:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985429376 00000100:00100000:2.0:1767320456.413399:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985429376 00000100:00100000:2.0:1767320456.413417:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985429760 00000100:00100000:2.0:1767320456.413419:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985429760 00000100:00100000:2.0:1767320456.413429:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430016 00000100:00100000:2.0:1767320456.413431:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430016 00000100:00100000:2.0:1767320456.413443:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5913 00000100:00100000:2.0:1767320456.413447:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985429376:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767320456.413494:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320456.413520:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5914 00000100:00100000:1.0:1767320456.413525:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985429760:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320456.413585:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430272 00000100:00100000:3.0:1767320456.413589:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430272 00000100:00100000:3.0:1767320456.413621:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430144 00000100:00100000:3.0:1767320456.413624:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430144 00000100:00100000:3.0:1767320456.413640:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3401 00000100:00100000:3.0:1767320456.413645:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985430272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320456.413692:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985430272:12345-0@lo:400:kworker.0 Request processed in 46us (1496us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320456.413701:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3401 00000100:00100000:3.0:1767320456.413714:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3402 00000100:00100000:3.0:1767320456.413719:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985430144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320456.413745:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985430144:12345-0@lo:400:kworker.0 Request processed in 26us (1489us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320456.413750:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3402 00000100:00080000:3.0:1767320456.413773:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320456.413786:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320456.413809:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985430144:0@lo:400:kworker.0 00000100:00100000:3.0:1767320456.413817:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985430272:0@lo:400:kworker.0 00000100:00100000:3.0:1767320456.413848:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5915 00000100:00100000:3.0:1767320456.413852:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985430016:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320456.414968:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320456.415036:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985429504:12345-0@lo:38:kworker.0 Request processed in 1875us (2002us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320456.415045:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1969 00000100:00080000:0.0:1767320456.415089:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320456.415094:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320456.415097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985429504:0@lo:38:kworker.0 00010000:00020000:2.0:1767320456.417731:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320456.421162:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985429376:12345-0@lo:38:kworker.0 Request processed in 7703us (8176us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320456.421200:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5913 00000100:00080000:0.0:1767320456.421307:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320456.421314:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320456.421318:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985429376:0@lo:38:kworker.0 00010000:00020000:1.0:1767320456.421993:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320456.422067:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985429760:12345-0@lo:38:kworker.0 Request processed in 8538us (8992us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320456.422076:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5914 00000100:00080000:0.0:1767320456.422320:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320456.422325:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320456.422329:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985429760:0@lo:38:kworker.0 00010000:00020000:3.0:1767320456.423254:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320456.423316:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985430016:12345-0@lo:38:kworker.0 Request processed in 9460us (10211us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320456.423326:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5915 00000100:00080000:0.0:1767320456.423349:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320456.423355:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320456.423359:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985430016:0@lo:38:kworker.0 00000100:00100000:1.0:1767320456.923668:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320456.923696:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985430400:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320456.923764:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320456.923929:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430400 00000100:00100000:2.0:1767320456.923939:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430400 00000100:00100000:2.0:1767320456.923999:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2602 00000100:00100000:2.0:1767320456.924010:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985430400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320456.924044:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320456.924102:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985430400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (337us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320456.924109:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2602 00000100:00100000:1.0:1767320456.924295:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985430400:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320456.924647:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985430528:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320456.924689:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320456.924833:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430528 00000100:00100000:3.0:1767320456.924840:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430528 00000100:00100000:3.0:1767320456.924880:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2603 00000100:00100000:3.0:1767320456.924905:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985430528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320456.924922:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320456.924960:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985430528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (269us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320456.924966:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2603 00000100:00100000:1.0:1767320456.925219:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985430528:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320457.057611:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320457.386480:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320457.386529:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320457.386628:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920618112 00000100:00100000:0.0:1767320457.386639:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920618112 00000100:00100000:0.0:1767320457.386708:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920617984 00000100:00100000:0.0:1767320457.386711:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920617984 00000100:00100000:0.0:1767320457.386742:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5233 00000100:00100000:0.0:1767320457.386754:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920618112:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320457.386848:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920618112:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (367us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320457.386856:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5233 00000100:00100000:0.0:1767320457.386876:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5916 00000100:00100000:0.0:1767320457.386881:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920617984:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320457.386927:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320457.386952:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920617984:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 69us (423us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320457.386959:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5916 00000100:00080000:2.0:1767320457.388030:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320458.077050:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320459.112353:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320460.123416:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320461.147932:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320461.534292:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320461.534303:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320461.534306:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320461.534310:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320461.534316:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320461.534324:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5704 00000100:00080000:0.0:1767320461.534330:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320461.534336:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320461.534379:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb480 x1853163985430656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320461.534403:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320461.534407:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320461.534409:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320461.534414:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5704 00000100:00080000:0.0:1767320461.534418:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320461.534419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985430656:0@lo:38:kworker.0 00000100:00080000:0.0:1767320461.534424:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320461.534441:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8a80 x1853163985430784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320461.534485:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320461.534489:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320461.534491:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320461.534496:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5704 00000100:00080000:0.0:1767320461.534501:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320461.534501:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320461.534508:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320461.534522:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9f80 x1853163985431040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320461.534540:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320461.534543:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320461.534545:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320461.534548:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5704 00000100:00080000:0.0:1767320461.534551:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320461.534555:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320461.534565:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4380 x1853163985431296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320461.534585:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985430784:0@lo:38:kworker.0 00000100:00100000:2.0:1767320461.534618:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320461.534632:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985431424:0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.534658:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985431040:0@lo:38:kworker.0 00000100:00100000:0.0:1767320461.534671:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320461.534736:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985430912:0@lo:400:kworker.0 00000100:00100000:3.0:1767320461.534853:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985431168:0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.535885:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320461.535921:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985431296:0@lo:38:kworker.0 00000100:00100000:2.0:1767320461.535957:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320461.536001:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430656 00000100:00100000:2.0:1767320461.536009:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430656 00000100:00100000:2.0:1767320461.536039:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431040 00000100:00100000:2.0:1767320461.536042:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431040 00000100:00100000:2.0:1767320461.536054:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431296 00000100:00100000:2.0:1767320461.536056:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431296 00000100:00100000:2.0:1767320461.536071:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5917 00000100:00100000:2.0:1767320461.536079:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985430656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320461.536101:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320461.536236:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985430656:12345-0@lo:38:kworker.0 Request processed in 155us (1738us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320461.536244:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5917 00000100:00100000:2.0:1767320461.536261:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5918 00000100:00100000:2.0:1767320461.536265:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985431040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320461.536275:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320461.536298:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985431040:12345-0@lo:38:kworker.0 Request processed in 31us (416us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320461.536304:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5918 00000100:00100000:2.0:1767320461.536316:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5919 00000100:00100000:2.0:1767320461.536323:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985431296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320461.536431:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320461.536457:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985431296:12345-0@lo:38:kworker.0 Request processed in 134us (499us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320461.536464:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5919 00000100:00080000:2.0:1767320461.536570:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320461.536576:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320461.536581:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985431040:0@lo:38:kworker.0 00000100:00080000:2.0:1767320461.536594:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320461.536597:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320461.536600:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985431296:0@lo:38:kworker.0 00000100:00080000:2.0:1767320461.536608:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320461.536611:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320461.536614:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985430656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320461.536684:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431424 00000100:00100000:2.0:1767320461.536688:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431424 00000100:00100000:2.0:1767320461.536730:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3403 00000100:00100000:2.0:1767320461.536738:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985431424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.536777:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985431424:12345-0@lo:400:kworker.0 Request processed in 41us (2107us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320461.536784:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3403 00000100:00100000:2.0:1767320461.536802:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430784 00000100:00100000:2.0:1767320461.536806:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430784 00000100:00100000:2.0:1767320461.536840:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1970 00000100:00100000:2.0:1767320461.536844:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985430784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320461.536855:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320461.536883:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985430784:12345-0@lo:38:kworker.0 Request processed in 38us (2266us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320461.536890:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1970 00000100:00080000:2.0:1767320461.536904:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320461.536920:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320461.536925:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320461.536929:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985430784:0@lo:38:kworker.0 00000100:00100000:0.0:1767320461.536954:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985431424:0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.536985:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985431552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.537011:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320461.537038:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431552 00000100:00100000:2.0:1767320461.537041:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431552 00000100:00100000:2.0:1767320461.537064:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3404 00000100:00100000:2.0:1767320461.537070:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985431552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.537110:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985431552:12345-0@lo:400:kworker.0 Request processed in 39us (97us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320461.537116:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3404 00000100:00080000:2.0:1767320461.537128:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320461.537142:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985431552:0@lo:400:kworker.0 00000100:00100000:1.0:1767320461.538240:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320461.538454:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320461.538639:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985430912 00000100:00100000:2.0:1767320461.538647:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985430912 00000100:00100000:2.0:1767320461.538702:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431168 00000100:00100000:2.0:1767320461.538705:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431168 00000100:00100000:2.0:1767320461.538729:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5234 00000100:00100000:2.0:1767320461.538737:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985430912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.538790:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985430912:12345-0@lo:400:kworker.0 Request processed in 54us (634us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320461.538802:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5234 00000100:00100000:2.0:1767320461.538827:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5235 00000100:00100000:2.0:1767320461.538834:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985431168:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320461.538864:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985431168:12345-0@lo:400:kworker.0 Request processed in 31us (411us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320461.538869:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5235 00000100:00100000:1.0:1767320461.539298:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985430912:0@lo:400:kworker.0 00000100:00080000:3.0:1767320461.542444:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320461.542509:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985431168:0@lo:400:kworker.0 00000100:00100000:0.0:1767320462.044556:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320462.044580:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985431680:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320462.044647:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320462.044770:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431680 00000100:00100000:2.0:1767320462.044777:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431680 00000100:00100000:2.0:1767320462.044843:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2604 00000100:00100000:2.0:1767320462.044854:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985431680:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320462.044886:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320462.044939:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985431680:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320462.044946:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2604 00000100:00100000:0.0:1767320462.045040:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985431680:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320462.046611:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320462.046627:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985431808:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320462.046690:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320462.046854:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431808 00000100:00100000:2.0:1767320462.046861:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431808 00000100:00100000:2.0:1767320462.046908:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2605 00000100:00100000:2.0:1767320462.046917:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985431808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320462.046940:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320462.047081:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985431808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 164us (390us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320462.047088:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2605 00000100:00100000:1.0:1767320462.047391:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985431808:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320462.171457:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320462.509387:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320462.509444:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320462.509584:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920618368 00000100:00100000:0.0:1767320462.509595:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920618368 00000100:00100000:0.0:1767320462.509673:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920618240 00000100:00100000:0.0:1767320462.509677:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920618240 00000100:00100000:0.0:1767320462.509700:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5236 00000100:00100000:0.0:1767320462.509710:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920618368:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320462.509786:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920618368:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (408us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320462.509808:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5236 00000100:00100000:0.0:1767320462.509828:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5920 00000100:00100000:0.0:1767320462.509832:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920618240:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320462.509854:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320462.509878:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920618240:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (434us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320462.509898:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5920 00000100:00080000:3.0:1767320462.512350:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320463.195369:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320464.219402:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320465.243377:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320466.268194:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320466.607124:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320466.610696:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920618752 00000100:00100000:2.0:1767320466.610714:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920618752 00000100:00100000:2.0:1767320466.610993:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5237 00000100:00100000:2.0:1767320466.611006:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920618752:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767320466.613566:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320466.613708:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920618752:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 2698us (6587us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320466.613914:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5237 00000400:00000100:3.0:1767320466.617771:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920618496 offset 0 length 520: 4 00000100:00100000:3.0:1767320466.617870:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320466.618343:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920618624 00000100:00100000:2.0:1767320466.618354:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920618624 00000100:00100000:2.0:1767320466.618404:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5921 00000100:00100000:2.0:1767320466.618413:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920618624:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320466.618438:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320466.618504:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920618624:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 90us (633us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320466.618512:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5921 00000100:00080000:0.0:1767320466.651361:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320466.651376:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320466.651381:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320466.651386:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320466.651391:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320466.651401:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5709 00000100:00100000:2.0:1767320466.651406:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320441/real 1767320441] req@ffff8ce1b5be8000 x1853163985425408/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320466 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320466.651407:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320466.651421:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320466.651451:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5be8000 x1853163985425408/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320466 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320466.651475:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320466.651482:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b800 x1853163985431936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320466.651502:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320466.651506:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320466.651508:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320466.651512:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5709 00000100:00080000:0.0:1767320466.651515:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320466.651522:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00080000:0.0:1767320466.651523:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320466.651538:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985425408:0@lo:250:kworker.0 00000100:00100000:0.0:1767320466.651554:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163985432064/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320466.651579:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985431936:0@lo:38:kworker.0 00000100:00080000:0.0:1767320466.651619:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320466.651625:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320466.651627:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320466.651632:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5709 00000100:00080000:0.0:1767320466.651634:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320466.651638:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320466.651654:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985432320/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320466.651672:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320466.651675:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320466.651677:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320466.651681:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5709 00000100:00080000:0.0:1767320466.651683:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320466.651686:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320466.651697:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74a80 x1853163985432576/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320466.651757:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985432704:0@lo:400:kworker.0 00000100:00100000:0.0:1767320466.651857:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320466.651918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320466.651977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985432064:0@lo:38:kworker.0 00000100:00100000:0.0:1767320466.652016:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985432832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320466.652073:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320466.652073:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320466.652128:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320466.652135:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320466.652137:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320466.652141:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5689 00000100:00080000:0.0:1767320466.652145:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320466.652149:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320466.652194:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75180 x1853163985432960/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320466.652291:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985432192:0@lo:400:kworker.0 00000100:00100000:1.0:1767320466.652338:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320466.652498:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985432320:0@lo:38:kworker.0 00000100:00100000:2.0:1767320466.652555:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985432448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.652587:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985432704 00000100:00100000:2.0:1767320466.652596:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985432704 00000100:00100000:2.0:1767320466.652649:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985432832 00000100:00100000:2.0:1767320466.652652:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985432832 00000100:00100000:2.0:1767320466.652680:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3405 00000100:00100000:2.0:1767320466.652689:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985432704:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.652758:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985432704:12345-0@lo:400:kworker.0 Request processed in 70us (901us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320466.652765:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3405 00000100:00100000:2.0:1767320466.652784:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3406 00000100:00100000:2.0:1767320466.652790:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985432832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.652826:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985432832:12345-0@lo:400:kworker.0 Request processed in 36us (765us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320466.652832:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3406 00000100:00100000:2.0:1767320466.652856:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985432064 00000100:00100000:2.0:1767320466.652860:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985432064 00000100:00100000:2.0:1767320466.652881:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1971 00000100:00100000:2.0:1767320466.652886:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985432064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320466.652905:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320466.652940:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985432064:12345-0@lo:38:kworker.0 Request processed in 52us (880us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320466.652945:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1971 00000100:00080000:2.0:1767320466.652968:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320466.652972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320466.654204:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985432832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320466.654252:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985432704:0@lo:400:kworker.0 00000100:00100000:1.0:1767320466.664451:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320466.665092:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320466.671183:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985431936 00000100:00100000:2.0:1767320466.671199:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985431936 00000100:00100000:2.0:1767320466.671275:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985432192 00000100:00100000:2.0:1767320466.671278:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985432192 00000100:00100000:2.0:1767320466.671314:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985432448 00000100:00100000:2.0:1767320466.671318:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985432448 00000100:00100000:2.0:1767320466.671341:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5238 00000100:00100000:2.0:1767320466.671351:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985432192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.671453:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985432192:12345-0@lo:400:kworker.0 Request processed in 103us (19114us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320466.671461:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5238 00000100:00100000:2.0:1767320466.671478:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5239 00000100:00100000:2.0:1767320466.671485:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985432448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.671524:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985432448:12345-0@lo:400:kworker.0 Request processed in 40us (7086us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320466.671531:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5239 00000100:00100000:2.0:1767320466.671543:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5922 00000100:00100000:2.0:1767320466.671547:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985431936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320466.671566:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320466.671595:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985431936:12345-0@lo:38:kworker.0 Request processed in 46us (19679us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320466.671601:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5922 00000100:00080000:2.0:1767320466.671623:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320466.671699:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985432448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.671747:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985432192:0@lo:400:kworker.0 00000100:00100000:3.0:1767320466.677224:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985432320 00000100:00100000:3.0:1767320466.677241:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985432320 00000100:00100000:1.0:1767320466.677249:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985432576:0@lo:38:kworker.0 00000100:00100000:3.0:1767320466.677297:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5923 00000100:00100000:3.0:1767320466.677310:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985432320:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320466.677325:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767320466.677339:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320466.677373:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985432576 00000100:00100000:1.0:1767320466.677380:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985432576 00000100:00100000:1.0:1767320466.677421:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5924 00000100:00100000:1.0:1767320466.677434:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985432576:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320466.677436:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985432320:12345-0@lo:38:kworker.0 Request processed in 122us (12339us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320466.677453:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5923 00010000:00020000:1.0:1767320466.677465:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320466.677522:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985432576:12345-0@lo:38:kworker.0 Request processed in 91us (195us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320466.677530:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5924 00000100:00080000:1.0:1767320466.677577:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320466.677582:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320466.677584:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985432576:0@lo:38:kworker.0 00000100:00100000:1.0:1767320466.677591:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985432960:0@lo:250:kworker.0 00000400:00000100:1.0:1767320466.677610:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985432960 offset 0 length 520: 4 00000100:00080000:1.0:1767320466.677642:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320466.677646:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320466.677649:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985431936:0@lo:38:kworker.0 00000100:00080000:1.0:1767320466.677661:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320466.677666:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320466.677669:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985432064:0@lo:38:kworker.0 00000100:00080000:1.0:1767320466.677709:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320466.677713:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320466.677716:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985432320:0@lo:38:kworker.0 00000100:00100000:0.0:1767320467.163727:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320467.163763:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985433088:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320467.163858:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320467.163974:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433088 00000100:00100000:3.0:1767320467.163986:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433088 00000100:00100000:3.0:1767320467.164052:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2606 00000100:00100000:3.0:1767320467.164063:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985433088:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320467.164111:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320467.164194:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985433088:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (306us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320467.164203:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2606 00000100:00100000:0.0:1767320467.164637:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985433088:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320467.164863:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320467.164877:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985433216:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320467.164939:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320467.165201:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433216 00000100:00100000:2.0:1767320467.165208:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433216 00000100:00100000:2.0:1767320467.165257:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2607 00000100:00100000:2.0:1767320467.165265:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985433216:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320467.165291:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320467.165343:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985433216:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (402us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320467.165351:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2607 00000100:00100000:1.0:1767320467.165575:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985433216:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320467.291771:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320468.315592:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320469.340486:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320470.366744:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320471.387468:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320471.725217:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320471.727045:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920618880 00000100:00100000:2.0:1767320471.727058:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920618880 00000100:00100000:2.0:1767320471.727196:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5925 00000100:00100000:2.0:1767320471.727204:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920618880:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320471.727227:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320471.727301:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920618880:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 97us (2095us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320471.727308:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5925 00000100:00100000:3.0:1767320471.730092:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320471.738003:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619008 00000100:00100000:0.0:1767320471.738019:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619008 00000100:00100000:2.0:1767320471.738179:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5240 00000100:00100000:2.0:1767320471.738194:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920619008:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320471.738280:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920619008:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (8196us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320471.738288:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5240 00000100:00080000:3.0:1767320471.738371:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320471.774326:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320471.774338:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320471.774341:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320471.774345:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320471.774350:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320471.774358:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5714 00000100:00080000:0.0:1767320471.774362:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320471.774370:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320471.774410:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74700 x1853163985433344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320471.774443:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320471.774446:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320471.774449:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320471.774452:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5714 00000100:00080000:0.0:1767320471.774455:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320471.774457:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320471.774470:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985433472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320471.774502:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320471.774505:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320471.774507:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320471.774510:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5714 00000100:00080000:0.0:1767320471.774512:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320471.774515:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320471.774526:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163985433728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320471.774538:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320471.774541:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320471.774543:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320471.774547:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5714 00000100:00080000:0.0:1767320471.774549:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320471.774553:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320471.774562:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985433984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320471.774574:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985433856:0@lo:400:kworker.0 00000100:00100000:0.0:1767320471.774595:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320471.774608:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985434112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320471.774663:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320471.774663:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320471.774737:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985434112 00000100:00100000:1.0:1767320471.774746:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985433344:0@lo:38:kworker.0 00000100:00100000:2.0:1767320471.774746:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985434112 00000100:00100000:2.0:1767320471.774834:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3407 00000100:00100000:2.0:1767320471.774844:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985434112:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320471.778084:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985434240:0@lo:400:kworker.0 00000100:00100000:0.0:1767320471.778132:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320471.778213:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433856 00000100:00100000:0.0:1767320471.778220:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433856 00000100:00100000:0.0:1767320471.779134:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5241 00000100:00100000:0.0:1767320471.779151:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985433856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320471.779224:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985433856:12345-0@lo:400:kworker.0 Request processed in 79us (4568us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320471.779229:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5241 00000100:00100000:1.0:1767320471.779325:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320471.780299:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985433856:0@lo:400:kworker.0 00000100:00100000:3.0:1767320471.780324:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320471.780334:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985433600:0@lo:400:kworker.0 00000100:00080000:3.0:1767320471.780376:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320471.780382:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320471.780410:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985434240 00000100:00100000:3.0:1767320471.780416:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985434240 00000100:00100000:3.0:1767320471.780464:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3408 00000100:00100000:3.0:1767320471.780474:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985434240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320471.781774:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985433472:0@lo:38:kworker.0 00000100:00100000:1.0:1767320471.781900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.781929:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985433728:0@lo:38:kworker.0 00000100:00100000:1.0:1767320471.781952:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320471.782083:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985434240:12345-0@lo:400:kworker.0 Request processed in 1609us (3948us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320471.782095:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3408 00000100:00080000:3.0:1767320471.782144:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320471.782186:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985434240:0@lo:400:kworker.0 00000100:00100000:3.0:1767320471.782221:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433472 00000100:00100000:3.0:1767320471.782227:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433472 00000100:00100000:3.0:1767320471.782254:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1972 00000100:00100000:3.0:1767320471.782259:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985433472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320471.782278:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320471.782312:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985433472:12345-0@lo:38:kworker.0 Request processed in 51us (413us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320471.782318:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1972 00000100:00100000:2.0:1767320471.782755:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985434112:12345-0@lo:400:kworker.0 Request processed in 7911us (8095us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320471.782763:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3407 00000100:00100000:3.0:1767320471.782828:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985434112:0@lo:400:kworker.0 00000100:00100000:1.0:1767320471.783892:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985433984:0@lo:38:kworker.0 00000100:00100000:1.0:1767320471.783929:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320471.785068:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433344 00000100:00100000:0.0:1767320471.785078:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433344 00000100:00100000:0.0:1767320471.785131:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433728 00000100:00100000:0.0:1767320471.785135:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433728 00000100:00100000:3.0:1767320471.785185:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5926 00000100:00100000:3.0:1767320471.785194:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985433344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320471.785218:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320471.785225:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5927 00000100:00100000:0.0:1767320471.785232:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985433728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320471.785244:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320471.785284:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985433344:12345-0@lo:38:kworker.0 Request processed in 88us (5958us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320471.785291:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5926 00000100:00100000:3.0:1767320471.785530:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985433728:12345-0@lo:38:kworker.0 Request processed in 296us (3575us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320471.785536:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5927 00000100:00100000:3.0:1767320471.785550:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433984 00000100:00100000:3.0:1767320471.785553:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433984 00000100:00100000:3.0:1767320471.785569:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5928 00000100:00100000:3.0:1767320471.785573:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985433984:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320471.785582:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320471.785605:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985433984:12345-0@lo:38:kworker.0 Request processed in 31us (1677us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320471.785611:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5928 00000100:00080000:1.0:1767320471.786035:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320471.786042:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.786046:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985433984:0@lo:38:kworker.0 00000100:00080000:1.0:1767320471.786076:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320471.786080:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.786083:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985433344:0@lo:38:kworker.0 00000100:00080000:1.0:1767320471.786093:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320471.786096:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.786099:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985433472:0@lo:38:kworker.0 00000100:00080000:1.0:1767320471.786107:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320471.786111:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.786113:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985433728:0@lo:38:kworker.0 00000100:00100000:1.0:1767320471.786145:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320471.790717:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985433600 00000100:00100000:3.0:1767320471.790732:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985433600 00000100:00100000:3.0:1767320471.790911:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5242 00000100:00100000:3.0:1767320471.790921:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985433600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320471.791011:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985433600:12345-0@lo:400:kworker.0 Request processed in 92us (4865us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320471.791017:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5242 00000100:00100000:1.0:1767320471.791215:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985433600:0@lo:400:kworker.0 00000100:00080000:2.0:1767320471.796058:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320472.284122:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985434368:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320472.284206:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320472.284252:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985434368 00000100:00100000:3.0:1767320472.284260:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985434368 00000100:00100000:3.0:1767320472.284314:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2608 00000100:00100000:3.0:1767320472.284322:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985434368:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320472.284359:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320472.284407:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985434368:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (201us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320472.284425:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2608 00000100:00100000:0.0:1767320472.284444:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985434368:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320472.284876:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985434496:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320472.284906:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320472.284979:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985434496 00000100:00100000:0.0:1767320472.284983:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985434496 00000100:00100000:0.0:1767320472.285008:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2609 00000100:00100000:0.0:1767320472.285012:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985434496:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320472.285024:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320472.285220:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985434496:12345-0@lo:13:osp-pre-0-1.0 Request processed in 206us (313us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320472.285225:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2609 00000100:00100000:1.0:1767320472.285281:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985434496:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320472.413996:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320473.435396:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320474.459869:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320475.483402:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320476.507386:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320476.845912:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320476.849937:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619264 00000100:00100000:3.0:1767320476.849952:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619264 00000100:00100000:3.0:1767320476.850041:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5243 00000100:00100000:3.0:1767320476.850053:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920619264:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767320476.850185:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320476.850927:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920619264:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 874us (5021us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320476.850936:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5243 00000100:00100000:2.0:1767320476.851138:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320476.851718:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619136 00000100:00100000:3.0:1767320476.851726:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619136 00000100:00100000:3.0:1767320476.851908:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5929 00000100:00100000:3.0:1767320476.851915:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920619136:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320476.851937:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320476.851991:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920619136:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 75us (853us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320476.851998:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5929 00000100:00080000:3.0:1767320476.894308:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320476.894318:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320476.894321:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320476.894324:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320476.894328:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320476.894337:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5720 00000100:00080000:3.0:1767320476.894342:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320476.894349:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320476.894385:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bea300 x1853163985434624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320476.894416:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320476.894420:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320476.894422:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320476.894425:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5720 00000100:00080000:3.0:1767320476.894427:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320476.894431:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320476.894443:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9500 x1853163985434752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320476.894473:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320476.894477:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320476.894479:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320476.894482:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5720 00000100:00080000:3.0:1767320476.894485:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320476.894488:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320476.894499:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7100 x1853163985435008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320476.894514:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320476.894517:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320476.894518:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320476.894522:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5720 00000100:00080000:3.0:1767320476.894524:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320476.894527:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320476.894566:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7d880 x1853163985435264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320476.894739:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985435136:0@lo:400:kworker.0 00000100:00100000:3.0:1767320476.894800:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320476.894924:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985434880:0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.894952:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320476.895037:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435136 00000100:00100000:1.0:1767320476.895046:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435136 00000100:00100000:1.0:1767320476.895096:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985434880 00000100:00100000:1.0:1767320476.895099:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985434880 00000100:00100000:1.0:1767320476.895121:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5244 00000100:00100000:1.0:1767320476.895130:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985435136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320476.895200:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985435136:12345-0@lo:400:kworker.0 Request processed in 72us (400us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320476.895207:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5244 00000100:00100000:1.0:1767320476.895222:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5245 00000100:00100000:1.0:1767320476.895228:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985434880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320476.895254:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985434880:12345-0@lo:400:kworker.0 Request processed in 26us (300us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320476.895259:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5245 00000100:00100000:1.0:1767320476.895281:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985435520:0@lo:400:kworker.0 00000100:00100000:1.0:1767320476.895307:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320476.895370:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985434624:0@lo:38:kworker.0 00000100:00100000:2.0:1767320476.895398:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320476.895720:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985435136:0@lo:400:kworker.0 00000100:00100000:3.0:1767320476.895749:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985435392:0@lo:400:kworker.0 00000100:00100000:3.0:1767320476.895773:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320476.896380:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985434624 00000100:00100000:3.0:1767320476.896387:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985434624 00000100:00100000:3.0:1767320476.896414:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5930 00000100:00100000:3.0:1767320476.896420:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985434624:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320476.896438:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320476.896473:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985434624:12345-0@lo:38:kworker.0 Request processed in 52us (1074us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320476.896480:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5930 00000100:00100000:3.0:1767320476.896498:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435520 00000100:00100000:3.0:1767320476.896501:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435520 00000100:00100000:3.0:1767320476.896539:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435392 00000100:00100000:3.0:1767320476.896542:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435392 00000100:00100000:3.0:1767320476.896565:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3409 00000100:00100000:3.0:1767320476.896571:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985435520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320476.896611:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985435520:12345-0@lo:400:kworker.0 Request processed in 31us (1293us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320476.896618:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3409 00000100:00100000:3.0:1767320476.896630:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3410 00000100:00100000:3.0:1767320476.896636:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985435392:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320476.896665:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985435392:12345-0@lo:400:kworker.0 Request processed in 29us (891us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320476.896671:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3410 00000100:00100000:3.0:1767320476.896699:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985435392:0@lo:400:kworker.0 00000100:00080000:3.0:1767320476.896892:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320476.896897:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320476.896899:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320476.897495:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985434880:0@lo:400:kworker.0 00000100:00100000:1.0:1767320476.897758:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985435520:0@lo:400:kworker.0 00000100:00080000:2.0:1767320476.897909:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320476.897914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320476.897918:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985434624:0@lo:38:kworker.0 00000100:00100000:2.0:1767320476.897926:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985434752:0@lo:38:kworker.0 00000100:00100000:2.0:1767320476.897966:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320476.898000:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985435008:0@lo:38:kworker.0 00000100:00100000:2.0:1767320476.898030:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320476.898054:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985435264:0@lo:38:kworker.0 00000100:00100000:2.0:1767320476.898075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320476.898159:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435008 00000100:00100000:1.0:1767320476.898195:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435008 00000100:00100000:1.0:1767320476.898219:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435264 00000100:00100000:1.0:1767320476.898222:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435264 00000100:00100000:1.0:1767320476.898238:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5931 00000100:00100000:1.0:1767320476.898243:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985435008:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320476.898258:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320476.898291:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985435008:12345-0@lo:38:kworker.0 Request processed in 46us (260us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320476.898297:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5931 00000100:00100000:1.0:1767320476.898311:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5932 00000100:00100000:1.0:1767320476.898315:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985435264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320476.898324:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320476.898345:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985435264:12345-0@lo:38:kworker.0 Request processed in 29us (269us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320476.898351:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5932 00000100:00100000:3.0:1767320476.898537:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985434752 00000100:00100000:3.0:1767320476.898542:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985434752 00000100:00100000:3.0:1767320476.898568:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1973 00000100:00100000:3.0:1767320476.898573:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985434752:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320476.898584:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320476.898616:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985434752:12345-0@lo:38:kworker.0 Request processed in 42us (650us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320476.898622:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1973 00000100:00080000:2.0:1767320476.898895:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320476.898900:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320476.898903:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985434752:0@lo:38:kworker.0 00000100:00080000:2.0:1767320476.898911:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320476.898914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320476.898916:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985435008:0@lo:38:kworker.0 00000100:00080000:2.0:1767320476.898925:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320476.898928:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320476.898930:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985435264:0@lo:38:kworker.0 00000100:00100000:0.0:1767320477.403414:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985435648:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320477.403495:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320477.404191:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435648 00000100:00100000:2.0:1767320477.404200:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435648 00000100:00100000:1.0:1767320477.404242:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985435776:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320477.404260:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2610 00000100:00100000:2.0:1767320477.404268:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985435648:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320477.404275:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:2.0:1767320477.404299:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320477.404307:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435776 00000100:00100000:1.0:1767320477.404310:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435776 00000100:00100000:2.0:1767320477.404338:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985435648:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (843us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320477.404340:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2611 00000100:00100000:2.0:1767320477.404345:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2610 00000100:00100000:1.0:1767320477.404347:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985435776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320477.404361:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320477.404388:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985435776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (112us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320477.404394:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2611 00000100:00100000:2.0:1767320477.404410:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985435776:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320477.404495:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985435648:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320477.532678:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320478.560125:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320479.579503:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320480.603559:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320481.627393:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320481.964088:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320481.964137:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320481.964322:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619392 00000100:00100000:3.0:1767320481.964342:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619392 00000100:00100000:3.0:1767320481.964390:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619520 00000100:00100000:3.0:1767320481.964394:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619520 00000100:00100000:3.0:1767320481.964463:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5246 00000100:00100000:3.0:1767320481.964476:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920619520:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320481.964550:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920619520:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (411us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320481.964561:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5246 00000100:00100000:3.0:1767320481.964580:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5933 00000100:00100000:3.0:1767320481.964585:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920619392:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320481.964605:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320481.964630:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920619392:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (553us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320481.964637:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5933 00000100:00080000:3.0:1767320481.964653:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320482.011580:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320482.011588:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320482.011590:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320482.011593:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320482.011597:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320482.011603:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5725 00000100:00080000:0.0:1767320482.011615:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320482.011621:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320482.011653:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74700 x1853163985435904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320482.011672:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320482.011675:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320482.011677:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320482.011681:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5725 00000100:00080000:0.0:1767320482.011683:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320482.011687:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320482.011702:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985436032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320482.011704:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985435904:0@lo:38:kworker.0 00000100:00080000:0.0:1767320482.011743:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320482.011746:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320482.011748:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320482.011751:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5725 00000100:00080000:0.0:1767320482.011753:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320482.011756:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320482.011765:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320482.011768:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985436288/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320482.011783:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320482.011785:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320482.011787:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320482.011789:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5725 00000100:00080000:0.0:1767320482.011791:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320482.011794:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320482.011803:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9500 x1853163985436544/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320482.011842:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985436672:0@lo:400:kworker.0 00000100:00100000:3.0:1767320482.011842:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985436160:0@lo:400:kworker.0 00000100:00100000:3.0:1767320482.011870:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320482.011872:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320482.011916:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985436032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320482.011940:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320482.011947:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985435904 00000100:00100000:1.0:1767320482.011953:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985435904 00000100:00100000:2.0:1767320482.011962:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985436288:0@lo:38:kworker.0 00000100:00100000:2.0:1767320482.011980:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320482.011987:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436160 00000100:00100000:1.0:1767320482.011989:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436160 00000100:00100000:2.0:1767320482.011993:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985436544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320482.012012:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320482.012028:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436288 00000100:00100000:1.0:1767320482.012031:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436288 00000100:00100000:2.0:1767320482.012036:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436544 00000100:00100000:2.0:1767320482.012038:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436544 00000100:00100000:1.0:1767320482.012044:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5247 00000100:00100000:1.0:1767320482.012054:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985436160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320482.012077:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5934 00000100:00100000:0.0:1767320482.012081:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320482.012082:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985435904:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320482.012087:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985436800:0@lo:400:kworker.0 00010000:00020000:2.0:1767320482.012099:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320482.012108:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320482.012134:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985435904:12345-0@lo:38:kworker.0 Request processed in 50us (368us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320482.012141:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5934 00000100:00100000:0.0:1767320482.012147:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436032 00000100:00100000:0.0:1767320482.012151:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436032 00000100:00100000:2.0:1767320482.012155:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5935 00000100:00100000:2.0:1767320482.012159:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985436288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320482.012186:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320482.012188:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1974 00000100:00100000:0.0:1767320482.012192:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985436032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320482.012201:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320482.012209:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985436288:12345-0@lo:38:kworker.0 Request processed in 48us (228us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320482.012215:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5935 00000100:00100000:0.0:1767320482.012219:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985436032:12345-0@lo:38:kworker.0 Request processed in 26us (279us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320482.012224:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5936 00000100:00100000:0.0:1767320482.012225:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1974 00000100:00100000:2.0:1767320482.012228:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985436544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320482.012236:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320482.012241:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985436416:0@lo:400:kworker.0 00000100:00100000:2.0:1767320482.012255:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985436544:12345-0@lo:38:kworker.0 Request processed in 26us (242us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320482.012260:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320482.012261:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5936 00000100:00100000:2.0:1767320482.012271:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436416 00000100:00100000:2.0:1767320482.012274:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436416 00000100:00080000:0.0:1767320482.012289:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320482.012295:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5248 00000100:00100000:2.0:1767320482.012300:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985436416:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320482.012331:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436672 00000100:00100000:0.0:1767320482.012335:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436672 00000100:00100000:2.0:1767320482.012347:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985436416:12345-0@lo:400:kworker.0 Request processed in 47us (86us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320482.012353:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5248 00000100:00100000:0.0:1767320482.012361:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436800 00000100:00100000:0.0:1767320482.012364:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436800 00000100:00100000:0.0:1767320482.012381:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3411 00000100:00080000:2.0:1767320482.012381:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320482.012385:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320482.012387:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985436672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320482.012389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985436032:0@lo:38:kworker.0 00000100:00080000:2.0:1767320482.012397:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320482.012400:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320482.012403:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985436288:0@lo:38:kworker.0 00000100:00080000:2.0:1767320482.012410:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320482.012413:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320482.012427:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985436544:0@lo:38:kworker.0 00000100:00100000:0.0:1767320482.012430:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985436672:12345-0@lo:400:kworker.0 Request processed in 44us (557us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320482.012448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320482.012451:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320482.012454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985435904:0@lo:38:kworker.0 00000100:00100000:0.0:1767320482.012461:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3411 00000100:00100000:0.0:1767320482.012472:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3412 00000100:00100000:0.0:1767320482.012478:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985436800:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320482.012487:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985436416:0@lo:400:kworker.0 00000100:00100000:0.0:1767320482.012503:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985436800:12345-0@lo:400:kworker.0 Request processed in 26us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320482.012508:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3412 00000100:00080000:0.0:1767320482.012526:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320482.012529:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320482.012543:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985436800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320482.012551:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985436672:0@lo:400:kworker.0 00000100:00100000:1.0:1767320482.012637:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985436160:12345-0@lo:400:kworker.0 Request processed in 585us (765us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320482.012644:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5247 00000100:00100000:3.0:1767320482.012716:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985436160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320482.524669:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320482.524695:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985436928:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320482.524761:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320482.524855:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985436928 00000100:00100000:2.0:1767320482.524864:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985436928 00000100:00100000:2.0:1767320482.524915:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2612 00000100:00100000:2.0:1767320482.524924:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985436928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320482.524951:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320482.524992:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985436928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 69us (231us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320482.525000:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2612 00000100:00100000:1.0:1767320482.525219:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985436928:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320482.525569:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320482.525580:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985437056:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320482.525611:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320482.525913:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437056 00000100:00100000:2.0:1767320482.525918:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437056 00000100:00100000:2.0:1767320482.525951:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2613 00000100:00100000:2.0:1767320482.525958:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985437056:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320482.525977:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320482.526016:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985437056:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (404us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320482.526022:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2613 00000100:00100000:0.0:1767320482.526124:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985437056:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320482.654816:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320483.676054:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320484.699396:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320485.723348:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320486.747839:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320487.087559:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320487.087618:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320487.091669:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619776 00000100:00100000:2.0:1767320487.091683:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619776 00000100:00100000:2.0:1767320487.091850:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619648 00000100:00100000:2.0:1767320487.091854:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619648 00000100:00100000:2.0:1767320487.091885:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5249 00000100:00100000:2.0:1767320487.091895:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920619776:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320487.091986:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920619776:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (4439us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320487.091995:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5249 00000100:00100000:2.0:1767320487.092015:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5937 00000100:00100000:2.0:1767320487.092020:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920619648:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320487.092055:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320487.092078:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920619648:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 57us (4461us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320487.092084:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5937 00000100:00080000:1.0:1767320487.100875:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320487.131423:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320487.131433:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320487.131437:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320487.131441:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320487.131446:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320487.131453:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320487.131458:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320487.131465:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320487.131505:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7dc00 x1853163985437184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320487.131534:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320487.131537:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320487.131539:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320487.131543:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320487.131545:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320487.131548:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320487.131560:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7f800 x1853163985437312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320487.131618:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320487.131622:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320487.131623:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320487.131627:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320487.131629:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320487.131633:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320487.131648:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ea00 x1853163985437568/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320487.131666:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320487.131669:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320487.131671:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320487.131674:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320487.131676:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320487.131679:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320487.131700:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7df80 x1853163985437824/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320487.131761:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985437952:0@lo:400:kworker.0 00000100:00100000:3.0:1767320487.131826:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320487.131925:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320487.131932:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985438080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320487.131954:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320487.132004:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437952 00000100:00100000:3.0:1767320487.132012:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437952 00000100:00100000:3.0:1767320487.132062:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985438080 00000100:00100000:3.0:1767320487.132066:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985438080 00000100:00100000:3.0:1767320487.132085:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3413 00000100:00100000:3.0:1767320487.132092:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985437952:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320487.132138:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985437952:12345-0@lo:400:kworker.0 Request processed in 46us (312us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320487.132145:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3413 00000100:00100000:2.0:1767320487.132149:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985437184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320487.132186:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3414 00000100:00100000:3.0:1767320487.132193:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985438080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320487.132214:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320487.132241:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985438080:12345-0@lo:400:kworker.0 Request processed in 47us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320487.132246:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985437312:0@lo:38:kworker.0 00000100:00100000:3.0:1767320487.132250:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3414 00000100:00100000:2.0:1767320487.132277:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320487.132298:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985438080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320487.132316:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985437952:0@lo:400:kworker.0 00000100:00100000:2.0:1767320487.132323:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985437568:0@lo:38:kworker.0 00000100:00080000:3.0:1767320487.132339:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320487.132357:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320487.132357:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320487.132379:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985437824:0@lo:38:kworker.0 00000100:00100000:3.0:1767320487.132387:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437312 00000100:00100000:3.0:1767320487.132392:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437312 00000100:00100000:2.0:1767320487.132411:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320487.132421:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1975 00000100:00100000:3.0:1767320487.132427:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985437312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320487.132444:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320487.132446:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437184 00000100:00100000:2.0:1767320487.132451:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437184 00000100:00100000:2.0:1767320487.132473:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437568 00000100:00100000:2.0:1767320487.132476:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437568 00000100:00100000:3.0:1767320487.132483:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985437312:12345-0@lo:38:kworker.0 Request processed in 55us (206us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320487.132491:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1975 00000100:00100000:2.0:1767320487.132494:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437824 00000100:00100000:2.0:1767320487.132497:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437824 00000100:00100000:2.0:1767320487.132708:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5938 00000100:00100000:2.0:1767320487.132713:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985437184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320487.132724:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320487.132751:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985437184:12345-0@lo:38:kworker.0 Request processed in 37us (538us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320487.132757:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5938 00000100:00100000:2.0:1767320487.132781:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5939 00000100:00100000:2.0:1767320487.132784:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985437568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320487.132793:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320487.132817:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985437568:12345-0@lo:38:kworker.0 Request processed in 31us (460us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320487.132822:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5939 00000100:00100000:2.0:1767320487.132833:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5940 00000100:00100000:2.0:1767320487.132836:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985437824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320487.132850:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320487.132874:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985437824:12345-0@lo:38:kworker.0 Request processed in 35us (463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320487.132881:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5940 00000100:00100000:1.0:1767320487.132887:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985437696:0@lo:400:kworker.0 00000100:00100000:0.0:1767320487.132932:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985437440:0@lo:400:kworker.0 00000100:00100000:0.0:1767320487.132958:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320487.143154:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320487.145579:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437440 00000100:00100000:3.0:1767320487.145594:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437440 00000100:00100000:2.0:1767320487.145673:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5250 00000100:00100000:2.0:1767320487.145689:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985437440:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320487.145828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320487.145835:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320487.145839:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985437184:0@lo:38:kworker.0 00000100:00080000:1.0:1767320487.145851:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320487.145855:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320487.145858:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985437312:0@lo:38:kworker.0 00000100:00080000:1.0:1767320487.145870:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320487.145873:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320487.145876:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985437568:0@lo:38:kworker.0 00000100:00100000:2.0:1767320487.145889:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985437440:12345-0@lo:400:kworker.0 Request processed in 204us (12927us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320487.145891:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320487.145898:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320487.145901:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985437824:0@lo:38:kworker.0 00000100:00100000:1.0:1767320487.145973:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985437440:0@lo:400:kworker.0 00000100:00080000:1.0:1767320487.146012:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320487.146272:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5250 00000100:00100000:2.0:1767320487.146700:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985437696 00000100:00100000:2.0:1767320487.146707:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985437696 00000100:00100000:2.0:1767320487.146753:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5251 00000100:00100000:2.0:1767320487.146761:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985437696:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320487.146777:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320487.146824:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985437696:12345-0@lo:400:kworker.0 Request processed in 63us (3675us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320487.146832:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5251 00000100:00100000:0.0:1767320487.146870:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985437696:0@lo:400:kworker.0 00000100:00100000:0.0:1767320487.643585:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320487.643610:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985438208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320487.643677:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320487.643774:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985438208 00000100:00100000:3.0:1767320487.643783:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985438208 00000100:00100000:3.0:1767320487.643877:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2614 00000100:00100000:3.0:1767320487.643888:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985438208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320487.643915:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320487.643954:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985438208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (276us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320487.643962:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2614 00000100:00100000:1.0:1767320487.644030:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320487.644040:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985438336:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320487.644071:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320487.644190:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985438336 00000100:00100000:0.0:1767320487.644194:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985438208:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320487.644197:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985438336 00000100:00100000:2.0:1767320487.644235:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2615 00000100:00100000:2.0:1767320487.644243:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985438336:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320487.644261:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320487.644303:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985438336:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320487.644310:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2615 00000100:00100000:1.0:1767320487.644348:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985438336:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320487.771402:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320488.795427:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320489.819447:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320490.843379:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320491.227312:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320466/real 1767320466] req@ffff8ce19db75180 x1853163985432960/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320491 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320491.227347:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19db75180 x1853163985432960/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320491 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320491.227368:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.227391:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320491.227395:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985432960:0@lo:250:kworker.0 00000100:00080000:0.0:1767320491.228022:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320491.228029:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320491.228032:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320491.228037:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5714 00000100:00080000:0.0:1767320491.228041:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320491.228048:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320491.228080:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985438464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320491.228100:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320491.228104:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320491.228106:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320491.228109:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5735 00000100:00080000:0.0:1767320491.228111:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320491.228114:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320491.228125:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b480 x1853163985438592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320491.228133:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320491.228136:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320491.228137:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320491.228140:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5735 00000100:00080000:0.0:1767320491.228142:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320491.228145:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320491.228153:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519500 x1853163985438720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320491.228200:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320491.228204:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320491.228205:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320491.228209:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5735 00000100:00080000:0.0:1767320491.228211:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320491.228214:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320491.228223:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a680 x1853163985438976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320491.228235:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320491.228238:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320491.228239:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320491.228242:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5735 00000100:00080000:0.0:1767320491.228244:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320491.228247:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320491.228255:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518000 x1853163985439232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320491.228337:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985439488:0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.228395:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320491.228443:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320491.228448:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985439360:0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.228502:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320491.231384:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985438848:0@lo:400:kworker.0 00000100:00100000:2.0:1767320491.231466:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320491.231540:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320491.231548:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985439104:0@lo:400:kworker.0 00000100:00100000:2.0:1767320491.231570:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.231813:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985438464:0@lo:250:kworker.0 00000400:00000100:1.0:1767320491.231839:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985438464 offset 0 length 520: 4 00000100:00100000:1.0:1767320491.231859:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985438592:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.231882:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.231898:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985438720:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.231920:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.231936:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985438976:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.231953:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.231970:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985439232:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.231987:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.232022:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985438848 00000100:00100000:1.0:1767320491.232031:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985438848 00000100:00100000:1.0:1767320491.232120:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985439104 00000100:00100000:1.0:1767320491.232123:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985439104 00000100:00100000:1.0:1767320491.232137:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985438592 00000100:00100000:1.0:1767320491.232140:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985438592 00000100:00100000:1.0:1767320491.232148:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985438976 00000100:00100000:1.0:1767320491.232150:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985438976 00000100:00100000:1.0:1767320491.232157:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985439232 00000100:00100000:1.0:1767320491.232159:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985439232 00000100:00100000:1.0:1767320491.232205:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5252 00000100:00100000:1.0:1767320491.232213:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985438848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320491.232288:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985438848:12345-0@lo:400:kworker.0 Request processed in 76us (822us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320491.232296:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5252 00000100:00100000:1.0:1767320491.232315:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5253 00000100:00100000:1.0:1767320491.232321:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985439104:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320491.232344:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985439104:12345-0@lo:400:kworker.0 Request processed in 25us (774us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320491.232350:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5253 00000100:00100000:1.0:1767320491.232360:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5941 00000100:00100000:1.0:1767320491.232364:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985438592:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320491.232382:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320491.232410:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985438592:12345-0@lo:38:kworker.0 Request processed in 44us (527us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320491.232416:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5941 00000100:00100000:1.0:1767320491.232426:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5942 00000100:00100000:1.0:1767320491.232429:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985438976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320491.232438:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320491.232456:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985438976:12345-0@lo:38:kworker.0 Request processed in 26us (502us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320491.232465:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5942 00000100:00100000:1.0:1767320491.232474:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5943 00000100:00100000:1.0:1767320491.232478:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985439232:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320491.232486:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320491.232508:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985439232:12345-0@lo:38:kworker.0 Request processed in 29us (520us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320491.232514:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5943 00000100:00080000:1.0:1767320491.232534:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320491.232571:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.232577:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320491.232581:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985438592:0@lo:38:kworker.0 00000100:00080000:1.0:1767320491.232591:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.232594:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320491.232597:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985438976:0@lo:38:kworker.0 00000100:00080000:1.0:1767320491.232604:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.232607:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320491.232609:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985439232:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.232651:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985439104:0@lo:400:kworker.0 00000100:00100000:1.0:1767320491.232661:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985438848:0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.233065:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985439488 00000100:00100000:0.0:1767320491.233072:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985439488 00000100:00100000:0.0:1767320491.233105:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985439360 00000100:00100000:0.0:1767320491.233108:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985439360 00000100:00100000:3.0:1767320491.233116:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3415 00000100:00100000:3.0:1767320491.233124:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985439488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.233128:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3416 00000100:00100000:0.0:1767320491.233133:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985439360:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.233193:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985439360:12345-0@lo:400:kworker.0 Request processed in 59us (4689us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320491.233199:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3416 00000100:00100000:2.0:1767320491.233300:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985438720 00000100:00100000:2.0:1767320491.233304:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985438720 00000100:00080000:1.0:1767320491.233364:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320491.233369:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320491.233453:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1976 00000100:00100000:2.0:1767320491.233458:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985438720:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320491.233470:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320491.233504:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985438720:12345-0@lo:38:kworker.0 Request processed in 45us (1582us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320491.233509:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1976 00000100:00080000:2.0:1767320491.233536:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320491.233540:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320491.233544:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985438720:0@lo:38:kworker.0 00000100:00100000:3.0:1767320491.234832:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985439488:12345-0@lo:400:kworker.0 Request processed in 1703us (6430us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320491.234844:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3415 00000100:00100000:0.0:1767320491.238734:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985439360:0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.238778:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985439488:0@lo:400:kworker.0 00010000:00010000:2.0:1767320491.868971:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320492.202126:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320492.204447:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767320492.207306:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920620160 offset 0 length 520: 4 00000100:00100000:1.0:1767320492.209936:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920620032 00000100:00100000:1.0:1767320492.209953:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920620032 00000100:00100000:1.0:1767320492.210057:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920619904 00000100:00100000:1.0:1767320492.210062:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920619904 00000100:00100000:1.0:1767320492.210093:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5254 00000100:00100000:1.0:1767320492.210103:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920620032:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320492.210203:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920620032:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 101us (8087us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320492.210211:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5254 00000100:00100000:1.0:1767320492.210231:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5944 00000100:00100000:1.0:1767320492.210235:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920619904:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320492.210268:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320492.210293:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920619904:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (5855us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320492.210299:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5944 00000100:00080000:2.0:1767320492.210513:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320492.763543:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985439616:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320492.763613:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320492.764136:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985439744:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320492.764192:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320492.764941:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985439616 00000100:00100000:3.0:1767320492.764950:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985439616 00000100:00100000:3.0:1767320492.765104:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985439744 00000100:00100000:3.0:1767320492.765108:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985439744 00000100:00100000:2.0:1767320492.765146:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2616 00000100:00100000:2.0:1767320492.765159:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985439616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320492.765210:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320492.765288:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985439616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 133us (1675us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320492.765303:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2616 00000100:00100000:2.0:1767320492.765322:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2617 00000100:00100000:2.0:1767320492.765328:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985439744:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320492.765348:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320492.765379:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985439744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (1186us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320492.765443:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2617 00000100:00100000:0.0:1767320492.765517:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985439744:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320492.765559:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985439616:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320492.891382:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320493.915435:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320494.942242:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320495.964231:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320496.347428:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320496.347435:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320496.347438:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320496.347441:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320496.347444:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320496.347450:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5739 00000100:00080000:0.0:1767320496.347453:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320496.347458:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320496.347491:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22680 x1853163985439872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320496.347513:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320496.347516:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320496.347517:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320496.347519:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5739 00000100:00080000:0.0:1767320496.347520:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320496.347523:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320496.347530:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21500 x1853163985440000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320496.347560:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320496.347563:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320496.347564:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320496.347566:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5739 00000100:00080000:0.0:1767320496.347567:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320496.347569:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320496.347575:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21880 x1853163985440256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320496.347585:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320496.347587:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320496.347588:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320496.347590:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5739 00000100:00080000:0.0:1767320496.347591:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320496.347593:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320496.347598:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20000 x1853163985440512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320496.347632:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985440640:0@lo:400:kworker.0 00000100:00100000:0.0:1767320496.347684:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320496.347729:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320496.347733:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985440768:0@lo:400:kworker.0 00000100:00100000:0.0:1767320496.347745:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320496.347810:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985440128:0@lo:400:kworker.0 00000100:00100000:1.0:1767320496.347831:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320496.347857:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320496.347860:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985440384:0@lo:400:kworker.0 00000100:00100000:1.0:1767320496.347871:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320496.347890:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440640 00000100:00100000:1.0:1767320496.347895:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440640 00000100:00100000:1.0:1767320496.347930:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440768 00000100:00100000:1.0:1767320496.347932:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440768 00000100:00100000:1.0:1767320496.347945:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3417 00000100:00100000:1.0:1767320496.347950:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985440640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320496.348085:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440128 00000100:00100000:0.0:1767320496.348088:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440128 00000100:00100000:0.0:1767320496.348107:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440384 00000100:00100000:0.0:1767320496.348108:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440384 00000100:00100000:0.0:1767320496.348119:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5255 00000100:00100000:0.0:1767320496.348125:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985440128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320496.348159:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985440128:12345-0@lo:400:kworker.0 Request processed in 33us (326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320496.348196:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5255 00000100:00100000:0.0:1767320496.348212:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5256 00000100:00100000:0.0:1767320496.348218:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985440384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320496.348238:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985440384:12345-0@lo:400:kworker.0 Request processed in 20us (365us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320496.348241:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5256 00000100:00080000:2.0:1767320496.348329:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320496.348335:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320496.348337:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320496.348349:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985439872:0@lo:38:kworker.0 00000100:00100000:2.0:1767320496.348377:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320496.348438:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3418 00000100:00100000:3.0:1767320496.348445:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985440768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320496.348470:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985440768:12345-0@lo:400:kworker.0 Request processed in 26us (724us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320496.348474:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3418 00000100:00100000:1.0:1767320496.348536:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985440640:12345-0@lo:400:kworker.0 Request processed in 39us (306us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320496.348549:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3417 00000100:00100000:1.0:1767320496.348563:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985439872 00000100:00100000:1.0:1767320496.348566:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985439872 00000100:00100000:1.0:1767320496.348579:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5945 00000100:00100000:1.0:1767320496.348581:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985439872:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320496.348595:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320496.348615:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985439872:12345-0@lo:38:kworker.0 Request processed in 33us (239us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320496.348619:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5945 00000100:00100000:1.0:1767320496.348651:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985440384:0@lo:400:kworker.0 00000100:00100000:1.0:1767320496.348657:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985440128:0@lo:400:kworker.0 00000100:00080000:2.0:1767320496.353239:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320496.354452:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320496.354457:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985439872:0@lo:38:kworker.0 00000100:00100000:2.0:1767320496.354468:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985440000:0@lo:38:kworker.0 00000100:00100000:2.0:1767320496.354527:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320496.354563:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985440256:0@lo:38:kworker.0 00000100:00100000:2.0:1767320496.354579:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320496.354589:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985440512:0@lo:38:kworker.0 00000100:00100000:2.0:1767320496.354601:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320496.354645:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440000 00000100:00100000:2.0:1767320496.354650:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440000 00000100:00100000:2.0:1767320496.354677:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1977 00000100:00100000:2.0:1767320496.354682:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985440000:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320496.354698:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320496.354725:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985440000:12345-0@lo:38:kworker.0 Request processed in 43us (198us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320496.354730:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1977 00000100:00100000:2.0:1767320496.354746:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440256 00000100:00100000:2.0:1767320496.354749:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440256 00000100:00100000:2.0:1767320496.354760:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440512 00000100:00100000:2.0:1767320496.354761:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440512 00000100:00100000:2.0:1767320496.354769:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5946 00000100:00100000:2.0:1767320496.354772:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985440256:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320496.354778:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320496.354792:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985440256:12345-0@lo:38:kworker.0 Request processed in 19us (213us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320496.354796:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5946 00000100:00100000:2.0:1767320496.354802:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5947 00000100:00100000:2.0:1767320496.354804:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985440512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320496.354809:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320496.354818:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985440512:12345-0@lo:38:kworker.0 Request processed in 13us (217us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320496.354821:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5947 00000100:00080000:2.0:1767320496.354848:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320496.354851:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320496.354853:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985440000:0@lo:38:kworker.0 00000100:00080000:2.0:1767320496.354857:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320496.354859:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320496.354861:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985440256:0@lo:38:kworker.0 00000100:00080000:2.0:1767320496.354865:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320496.354867:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320496.354868:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985440512:0@lo:38:kworker.0 00000100:00100000:1.0:1767320496.357894:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985440768:0@lo:400:kworker.0 00000100:00100000:1.0:1767320496.357908:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985440640:0@lo:400:kworker.0 00010000:00010000:2.0:1767320496.987372:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320497.322890:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320497.324507:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920620416 00000100:00100000:0.0:1767320497.324520:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920620416 00000100:00100000:0.0:1767320497.324642:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5257 00000100:00100000:0.0:1767320497.324655:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920620416:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320497.324868:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920620416:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 216us (1989us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320497.324874:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5257 00000100:00080000:2.0:1767320497.324969:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320497.329216:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320497.329620:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920620288 00000100:00100000:0.0:1767320497.329633:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920620288 00000100:00100000:0.0:1767320497.329676:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5948 00000100:00100000:0.0:1767320497.329684:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920620288:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320497.329711:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320497.329833:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920620288:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 147us (628us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320497.329841:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5948 00000100:00100000:3.0:1767320497.884062:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985440896:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320497.884383:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320497.884429:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320497.884437:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985441024:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320497.884504:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985440896 00000100:00100000:1.0:1767320497.884511:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985440896 00000100:00100000:1.0:1767320497.884553:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2618 00000100:00100000:1.0:1767320497.884563:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985440896:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320497.884578:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:1.0:1767320497.884592:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320497.884642:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985440896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320497.884650:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2618 00000100:00100000:1.0:1767320497.884666:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441024 00000100:00100000:1.0:1767320497.884670:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441024 00000100:00100000:1.0:1767320497.884691:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2619 00000100:00100000:1.0:1767320497.884696:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985441024:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320497.884790:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985440896:0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320497.885409:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320497.885535:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985441024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 839us (1075us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320497.885541:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2619 00000100:00100000:3.0:1767320497.885853:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985441024:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320498.012467:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320499.035520:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320500.059892:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320501.083380:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320501.467782:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320501.467793:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320501.467796:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320501.467799:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320501.467803:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320501.467811:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5744 00000100:00080000:0.0:1767320501.467815:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320501.467828:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320501.467866:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20e00 x1853163985441152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320501.467898:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320501.467901:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320501.467903:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320501.467906:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5744 00000100:00080000:0.0:1767320501.467908:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320501.467911:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320501.467924:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20a80 x1853163985441280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320501.467960:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320501.467963:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320501.467965:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320501.467969:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5744 00000100:00080000:0.0:1767320501.467971:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320501.467974:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320501.467985:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20000 x1853163985441536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320501.467998:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320501.468001:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320501.468003:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320501.468006:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5744 00000100:00080000:0.0:1767320501.468008:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320501.468010:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320501.468017:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21500 x1853163985441792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320501.468071:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985442048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320501.468159:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320501.468230:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320501.468236:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985441920:0@lo:400:kworker.0 00000100:00100000:1.0:1767320501.468256:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442048 00000100:00100000:0.0:1767320501.468259:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320501.468265:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442048 00000100:00100000:0.0:1767320501.468289:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985441408:0@lo:400:kworker.0 00000100:00100000:0.0:1767320501.468311:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320501.468314:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441920 00000100:00100000:1.0:1767320501.468317:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441920 00000100:00100000:0.0:1767320501.468330:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320501.468336:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985441664:0@lo:400:kworker.0 00000100:00100000:1.0:1767320501.468336:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3419 00000100:00100000:1.0:1767320501.468344:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985442048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320501.468353:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320501.468374:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441408 00000100:00100000:0.0:1767320501.468378:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441408 00000100:00100000:1.0:1767320501.468396:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985442048:12345-0@lo:400:kworker.0 Request processed in 52us (236us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320501.468402:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441664 00000100:00100000:1.0:1767320501.468402:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3419 00000100:00100000:0.0:1767320501.468404:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441664 00000100:00100000:1.0:1767320501.468416:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3420 00000100:00100000:0.0:1767320501.468420:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5258 00000100:00100000:1.0:1767320501.468421:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985441920:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320501.468425:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985441408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320501.468448:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985441920:12345-0@lo:400:kworker.0 Request processed in 26us (188us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320501.468453:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3420 00000100:00080000:1.0:1767320501.468472:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320501.468476:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320501.468478:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320501.468506:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985441920:0@lo:400:kworker.0 00000100:00100000:1.0:1767320501.468517:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985442048:0@lo:400:kworker.0 00000100:00100000:1.0:1767320501.468549:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5259 00000100:00100000:1.0:1767320501.468555:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985441664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320501.468588:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985441664:12345-0@lo:400:kworker.0 Request processed in 34us (234us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320501.468595:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5259 00000100:00100000:1.0:1767320501.468620:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985441664:0@lo:400:kworker.0 00000100:00100000:0.0:1767320501.468681:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985441408:12345-0@lo:400:kworker.0 Request processed in 255us (368us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320501.468688:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5258 00000100:00100000:1.0:1767320501.468725:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985441408:0@lo:400:kworker.0 00000100:00100000:2.0:1767320501.468778:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985441152:0@lo:38:kworker.0 00000100:00100000:2.0:1767320501.468848:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320501.468863:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985441280:0@lo:38:kworker.0 00000100:00100000:2.0:1767320501.468896:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320501.468925:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985441536:0@lo:38:kworker.0 00000100:00100000:2.0:1767320501.468944:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320501.468945:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441280 00000100:00100000:1.0:1767320501.468949:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441280 00000100:00100000:2.0:1767320501.468959:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985441792:0@lo:38:kworker.0 00000100:00100000:1.0:1767320501.468969:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1978 00000100:00100000:1.0:1767320501.468973:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985441280:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320501.468977:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767320501.468991:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320501.469004:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441152 00000100:00100000:2.0:1767320501.469007:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441152 00000100:00100000:1.0:1767320501.469022:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985441280:12345-0@lo:38:kworker.0 Request processed in 47us (125us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320501.469022:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441536 00000100:00100000:2.0:1767320501.469025:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441536 00000100:00100000:1.0:1767320501.469028:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1978 00000100:00100000:2.0:1767320501.469035:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985441792 00000100:00100000:2.0:1767320501.469038:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985441792 00000100:00100000:2.0:1767320501.469047:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5949 00000100:00100000:2.0:1767320501.469051:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985441152:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320501.469054:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5950 00000100:00100000:1.0:1767320501.469057:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985441536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320501.469060:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320501.469065:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320501.469097:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985441152:12345-0@lo:38:kworker.0 Request processed in 44us (248us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320501.469099:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985441536:12345-0@lo:38:kworker.0 Request processed in 40us (154us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320501.469103:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5949 00000100:00100000:1.0:1767320501.469107:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5950 00000100:00100000:2.0:1767320501.469115:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5951 00000100:00100000:2.0:1767320501.469119:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985441792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320501.469128:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320501.469149:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985441792:12345-0@lo:38:kworker.0 Request processed in 29us (171us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320501.469155:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5951 00000100:00080000:2.0:1767320501.469203:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320501.469207:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320501.469211:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985441152:0@lo:38:kworker.0 00000100:00080000:2.0:1767320501.469219:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320501.469221:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320501.469223:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985441280:0@lo:38:kworker.0 00000100:00080000:2.0:1767320501.469227:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320501.469229:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320501.469230:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985441536:0@lo:38:kworker.0 00000100:00080000:2.0:1767320501.469235:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320501.469236:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320501.469238:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985441792:0@lo:38:kworker.0 00010000:00010000:1.0:1767320502.109211:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320502.447763:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320502.447864:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320502.456669:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920620672 00000100:00100000:1.0:1767320502.456669:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920620544 00000100:00100000:0.0:1767320502.456685:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920620672 00000100:00100000:1.0:1767320502.456685:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920620544 00000100:00100000:1.0:1767320502.456793:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5260 00000100:00100000:0.0:1767320502.456805:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5952 00000100:00100000:0.0:1767320502.456811:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920620544:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767320502.456812:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920620672:12345-192.168.204.26@tcp:400:kworker.0 00010000:00020000:0.0:1767320502.456834:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320502.456892:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920620544:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 80us (9027us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320502.456892:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920620672:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (9136us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320502.456900:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5260 00000100:00080000:2.0:1767320502.456947:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320502.457364:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5952 00000100:00100000:0.0:1767320503.003429:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985442304:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320503.003486:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320503.003528:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985442176:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320503.003528:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442304 00000100:00100000:1.0:1767320503.003536:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442304 00000100:00100000:0.0:1767320503.003550:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320503.003639:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442176 00000100:00100000:1.0:1767320503.003643:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442176 00000100:00080000:3.0:1767320503.003666:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320503.003668:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2620 00000100:00080000:3.0:1767320503.003670:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320503.003679:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985442304:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320503.003709:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320503.003847:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2621 00000100:00100000:1.0:1767320503.003854:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb3050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985442176:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320503.003862:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985442304:12345-0@lo:13:osp-pre-0-1.0 Request processed in 186us (374us total) trans 0 rc 0/0 00002000:00000024:1.0:1767320503.003869:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320503.003869:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2620 00000100:00100000:1.0:1767320503.003899:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb3050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985442176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320503.003905:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2621 00000100:00100000:0.0:1767320503.004013:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985442304:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320503.004024:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985442176:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320503.133561:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320504.156497:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320505.179417:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320506.208601:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320506.587819:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320506.587829:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320506.587833:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320506.587838:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320506.587843:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320506.587851:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5749 00000100:00080000:0.0:1767320506.587856:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320506.587862:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320506.587897:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3b80 x1853163985442432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320506.587928:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320506.587931:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320506.587933:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320506.587936:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5749 00000100:00080000:0.0:1767320506.587939:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320506.587942:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320506.587965:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3800 x1853163985442560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320506.588012:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320506.588016:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320506.588017:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320506.588021:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5749 00000100:00080000:0.0:1767320506.588023:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320506.588026:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320506.588039:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1500 x1853163985442816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320506.588102:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320506.588106:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320506.588108:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767320506.588113:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985442688:0@lo:400:kworker.0 00000100:00080000:0.0:1767320506.588114:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5749 00000100:00080000:0.0:1767320506.588119:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320506.588126:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320506.588140:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985443072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320506.588211:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320506.588215:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985442944:0@lo:400:kworker.0 00000100:00100000:2.0:1767320506.588290:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985442432:0@lo:38:kworker.0 00000100:00100000:1.0:1767320506.588862:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985443200:0@lo:400:kworker.0 00000100:00100000:0.0:1767320506.589334:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320506.589404:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442688 00000100:00100000:0.0:1767320506.589412:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442688 00000100:00100000:0.0:1767320506.589462:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442944 00000100:00100000:0.0:1767320506.589466:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442944 00000100:00100000:0.0:1767320506.589491:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5261 00000100:00100000:0.0:1767320506.589500:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985442688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320506.589549:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985442688:12345-0@lo:400:kworker.0 Request processed in 49us (1339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320506.589556:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5261 00000100:00100000:0.0:1767320506.589574:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5262 00000100:00100000:0.0:1767320506.589580:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985442944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320506.589610:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985442944:12345-0@lo:400:kworker.0 Request processed in 30us (276us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320506.589615:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5262 00000100:00100000:0.0:1767320506.592156:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985442944:0@lo:400:kworker.0 00000100:00100000:0.0:1767320506.592230:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985443328:0@lo:400:kworker.0 00000100:00100000:0.0:1767320506.592290:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320506.592365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320506.592398:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443328 00000100:00100000:0.0:1767320506.592404:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443328 00000100:00100000:0.0:1767320506.592458:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3421 00000100:00100000:0.0:1767320506.592466:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985443328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320506.592516:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985443328:12345-0@lo:400:kworker.0 Request processed in 50us (225us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320506.592523:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3421 00000100:00080000:0.0:1767320506.592540:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320506.592556:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985443328:0@lo:400:kworker.0 00000100:00100000:2.0:1767320506.593198:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320506.593199:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985442688:0@lo:400:kworker.0 00000100:00100000:2.0:1767320506.593227:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442432 00000100:00100000:2.0:1767320506.593232:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442432 00000100:00100000:2.0:1767320506.593253:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5953 00000100:00100000:2.0:1767320506.593259:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985442432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320506.593278:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320506.593307:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985442432:12345-0@lo:38:kworker.0 Request processed in 47us (110us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320506.593313:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5953 00000100:00080000:3.0:1767320506.593346:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320506.593352:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320506.593355:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985442432:0@lo:38:kworker.0 00000100:00100000:3.0:1767320506.593360:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985442560:0@lo:38:kworker.0 00000100:00100000:3.0:1767320506.593553:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320506.593568:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320506.593582:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985442816:0@lo:38:kworker.0 00000100:00100000:3.0:1767320506.593604:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320506.593614:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985443072:0@lo:38:kworker.0 00000100:00100000:3.0:1767320506.593626:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320506.598847:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442560 00000100:00100000:2.0:1767320506.598862:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442560 00000100:00100000:2.0:1767320506.598939:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1979 00000100:00100000:2.0:1767320506.598947:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985442560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320506.598980:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320506.599074:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985442560:12345-0@lo:38:kworker.0 Request processed in 127us (5520us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320506.599082:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1979 00000100:00100000:2.0:1767320506.599110:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985442816 00000100:00100000:2.0:1767320506.599114:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985442816 00000100:00100000:2.0:1767320506.599134:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443072 00000100:00100000:2.0:1767320506.599137:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443072 00000100:00100000:2.0:1767320506.599153:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5954 00000100:00100000:2.0:1767320506.599157:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985442816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320506.599196:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320506.599223:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985442816:12345-0@lo:38:kworker.0 Request processed in 64us (5618us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320506.599229:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5954 00000100:00100000:2.0:1767320506.599240:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5955 00000100:00100000:2.0:1767320506.599245:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985443072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320506.599253:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320506.599274:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985443072:12345-0@lo:38:kworker.0 Request processed in 28us (5646us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320506.599279:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5955 00000100:00080000:3.0:1767320506.599800:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320506.599807:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320506.599811:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985442560:0@lo:38:kworker.0 00000100:00080000:3.0:1767320506.599822:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320506.599826:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320506.599829:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985442816:0@lo:38:kworker.0 00000100:00080000:3.0:1767320506.599838:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320506.599841:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320506.599844:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985443072:0@lo:38:kworker.0 00000100:00100000:0.0:1767320506.600067:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443200 00000100:00100000:0.0:1767320506.600073:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443200 00000100:00100000:0.0:1767320506.600142:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3422 00000100:00100000:0.0:1767320506.600150:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985443200:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320506.600221:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985443200:12345-0@lo:400:kworker.0 Request processed in 72us (6653us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320506.600227:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3422 00000100:00100000:1.0:1767320506.600367:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985443200:0@lo:400:kworker.0 00000100:00080000:2.0:1767320506.600418:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320507.227555:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320507.562112:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320507.562440:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920620800 00000100:00100000:0.0:1767320507.562472:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920620800 00000100:00100000:0.0:1767320507.562552:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5956 00000100:00100000:0.0:1767320507.562560:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920620800:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320507.562583:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320507.562653:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920620800:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (552us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320507.562660:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5956 00000100:00100000:1.0:1767320507.563541:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320507.564129:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920620928 00000100:00100000:0.0:1767320507.564137:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920620928 00000100:00100000:0.0:1767320507.564507:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5263 00000100:00100000:0.0:1767320507.564518:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920620928:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320507.564576:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920620928:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 61us (1048us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320507.564583:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5263 00000100:00080000:2.0:1767320507.564658:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320508.123691:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985443456:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320508.123822:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320508.123936:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443456 00000100:00100000:2.0:1767320508.123944:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443456 00000100:00100000:2.0:1767320508.124023:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2622 00000100:00100000:2.0:1767320508.124034:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985443456:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320508.124073:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320508.124114:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985443456:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (293us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320508.124121:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2622 00000100:00080000:3.0:1767320508.124266:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320508.124299:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985443456:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320508.125277:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985443584:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320508.125319:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320508.125434:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443584 00000100:00100000:1.0:1767320508.125440:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443584 00000100:00100000:1.0:1767320508.125489:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2623 00000100:00100000:1.0:1767320508.125497:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985443584:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320508.125518:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320508.125561:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985443584:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (241us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320508.125567:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2623 00000100:00100000:0.0:1767320508.125633:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985443584:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320508.125695:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320508.251975:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320509.275427:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320510.299402:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320511.327407:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320511.712946:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320511.712957:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320511.712960:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320511.712964:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320511.712968:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320511.712976:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5754 00000100:00080000:2.0:1767320511.712980:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320511.712986:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320511.713036:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76d80 x1853163985443712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320511.713069:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320511.713072:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320511.713141:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320511.713146:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5754 00000100:00080000:2.0:1767320511.713149:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320511.713153:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320511.713272:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985443840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320511.713371:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985443968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320511.713462:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.713613:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985443712:0@lo:38:kworker.0 00000100:00100000:3.0:1767320511.713645:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.713673:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985443840:0@lo:38:kworker.0 00000100:00100000:3.0:1767320511.713695:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.713735:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443840 00000100:00100000:3.0:1767320511.713743:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443840 00000100:00100000:3.0:1767320511.713783:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1980 00000100:00100000:3.0:1767320511.713791:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985443840:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320511.713825:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320511.713861:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985443840:12345-0@lo:38:kworker.0 Request processed in 69us (165us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320511.713870:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1980 00000100:00100000:0.0:1767320511.713883:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443968 00000100:00100000:0.0:1767320511.713897:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443968 00000100:00080000:3.0:1767320511.713934:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320511.713940:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320511.713944:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985443840:0@lo:38:kworker.0 00000100:00100000:0.0:1767320511.713953:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985443712 00000100:00080000:3.0:1767320511.713955:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320511.713958:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985443712 00000100:00100000:0.0:1767320511.713982:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5264 00000100:00100000:0.0:1767320511.713991:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985443968:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.714004:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5957 00000100:00100000:3.0:1767320511.714011:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985443712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320511.714023:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320511.714053:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320511.714058:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320511.714060:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320511.714064:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5754 00000100:00080000:2.0:1767320511.714067:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320511.714070:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320511.714096:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75180 x1853163985444096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320511.714122:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320511.714125:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320511.714127:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320511.714130:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5754 00000100:00080000:2.0:1767320511.714134:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320511.714137:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320511.714148:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74a80 x1853163985444352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320511.715401:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985443712:12345-0@lo:38:kworker.0 Request processed in 1383us (1749us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320511.715413:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5957 00000100:00100000:3.0:1767320511.715451:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985444480:0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.715531:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.715568:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444480 00000100:00100000:3.0:1767320511.715576:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444480 00000100:00100000:3.0:1767320511.715628:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3423 00000100:00100000:3.0:1767320511.715637:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985444480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.715676:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985444480:12345-0@lo:400:kworker.0 Request processed in 40us (145us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320511.715682:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3423 00000100:00080000:3.0:1767320511.715725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320511.715758:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985444480:0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.715777:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320511.715784:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985444224:0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.715818:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.715836:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444224 00000100:00100000:3.0:1767320511.715840:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444224 00000100:00100000:3.0:1767320511.715868:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5265 00000100:00100000:3.0:1767320511.715874:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985444224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.715912:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985444224:12345-0@lo:400:kworker.0 Request processed in 38us (93us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320511.715919:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5265 00000100:00080000:3.0:1767320511.715932:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320511.715946:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985444224:0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.715965:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985444096:0@lo:38:kworker.0 00000100:00100000:3.0:1767320511.715990:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.716154:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444096 00000100:00100000:3.0:1767320511.716159:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444096 00000100:00100000:3.0:1767320511.716196:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5958 00000100:00100000:3.0:1767320511.716201:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985444096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320511.716217:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320511.716243:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985444096:12345-0@lo:38:kworker.0 Request processed in 41us (252us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320511.716250:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5958 00000100:00080000:3.0:1767320511.716273:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320511.716278:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320511.716281:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985444096:0@lo:38:kworker.0 00000100:00100000:3.0:1767320511.716288:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985444352:0@lo:38:kworker.0 00000100:00100000:3.0:1767320511.716309:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.716325:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444352 00000100:00100000:3.0:1767320511.716329:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444352 00000100:00100000:3.0:1767320511.716345:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5959 00000100:00100000:3.0:1767320511.716349:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985444352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320511.716358:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320511.716380:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985444352:12345-0@lo:38:kworker.0 Request processed in 30us (71us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320511.716386:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5959 00000100:00080000:3.0:1767320511.716406:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320511.716410:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320511.716413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985444352:0@lo:38:kworker.0 00000100:00080000:3.0:1767320511.716423:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320511.716427:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320511.716429:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985443712:0@lo:38:kworker.0 00000100:00100000:3.0:1767320511.716484:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985444608:0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.716509:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320511.716538:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444608 00000100:00100000:3.0:1767320511.716541:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444608 00000100:00100000:3.0:1767320511.716566:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3424 00000100:00100000:3.0:1767320511.716572:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985444608:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320511.716601:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985444608:12345-0@lo:400:kworker.0 Request processed in 29us (93us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320511.716608:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3424 00000100:00100000:3.0:1767320511.716630:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985444608:0@lo:400:kworker.0 00000100:00080000:3.0:1767320511.716641:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320511.722452:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985443968:12345-0@lo:400:kworker.0 Request processed in 8447us (8974us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320511.722469:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5264 00000100:00100000:1.0:1767320511.723131:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985443968:0@lo:400:kworker.0 00010000:00010000:1.0:1767320512.347911:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320512.682332:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320512.682432:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920621184 00000100:00100000:0.0:1767320512.682441:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920621184 00000100:00100000:0.0:1767320512.682501:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5266 00000100:00100000:0.0:1767320512.682518:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920621184:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320512.682601:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920621184:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (276us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320512.682608:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320512.682613:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5266 00000100:00100000:3.0:1767320512.694849:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320512.694940:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920621056 00000100:00100000:1.0:1767320512.694950:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920621056 00000100:00100000:1.0:1767320512.695006:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5960 00000100:00100000:1.0:1767320512.695014:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920621056:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320512.695037:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320512.695103:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920621056:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 87us (263us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320512.695110:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5960 00000100:00100000:3.0:1767320513.243590:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320513.243616:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985444736:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320513.243690:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320513.243823:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444736 00000100:00100000:2.0:1767320513.243832:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444736 00000100:00100000:2.0:1767320513.243915:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2624 00000100:00100000:2.0:1767320513.243927:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985444736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320513.243977:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320513.244033:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985444736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (341us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320513.244041:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2624 00000100:00080000:3.0:1767320513.244144:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320513.244342:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985444736:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320513.244545:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320513.244557:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985444864:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320513.244590:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320513.244669:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444864 00000100:00100000:1.0:1767320513.244674:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444864 00000100:00100000:1.0:1767320513.244716:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2625 00000100:00100000:1.0:1767320513.244723:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985444864:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320513.244743:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320513.244782:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985444864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (191us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320513.244788:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2625 00000100:00100000:1.0:1767320513.244815:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985444864:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320513.244884:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320513.371653:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320514.395393:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320515.419859:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320516.443501:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320516.779687:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767320516.779723:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920621312 offset 0 length 520: 4 00000100:00100000:2.0:1767320516.779744:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320516.779804:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920621568 00000100:00100000:0.0:1767320516.779814:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920621568 00000100:00100000:1.0:1767320516.779818:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920621440 00000100:00100000:1.0:1767320516.779822:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920621440 00000100:00100000:1.0:1767320516.779871:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5961 00000100:00100000:1.0:1767320516.779879:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920621440:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320516.779954:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320516.780015:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920621440:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 137us (270us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320516.780022:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5961 00000100:00100000:0.0:1767320516.780248:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5267 00000100:00100000:0.0:1767320516.780256:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920621568:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320516.780321:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920621568:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 65us (642us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320516.780327:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5267 00000100:00080000:2.0:1767320516.785087:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320516.827276:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.827287:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320516.827291:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.827294:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.827298:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320516.827315:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5759 00000100:00100000:1.0:1767320516.827316:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320491/real 1767320491] req@ffff8ce1b351a300 x1853163985438464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320516 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320516.827320:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.827330:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320516.827364:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b351a300 x1853163985438464/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320516 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320516.827370:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74a80 x1853163985444992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320516.827397:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.827401:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.827402:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320516.827404:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985444992:0@lo:38:kworker.0 00000100:00080000:2.0:1767320516.827409:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5759 00000100:00080000:2.0:1767320516.827413:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.827417:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320516.827430:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75180 x1853163985445120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320516.827462:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.827465:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.827467:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320516.827472:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320516.827485:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5759 00000100:00080000:2.0:1767320516.827488:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.827491:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320516.827501:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77100 x1853163985445376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320516.827505:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985445248:0@lo:400:kworker.0 00000100:00080000:2.0:1767320516.827525:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320516.827526:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320516.827531:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.827534:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320516.827537:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5759 00000100:00080000:1.0:1767320516.827540:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00080000:2.0:1767320516.827541:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.827546:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320516.827549:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985438464:0@lo:250:kworker.0 00000100:00100000:3.0:1767320516.827550:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320516.827568:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938380 x1853163985445632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320516.827591:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985445120:0@lo:38:kworker.0 00000100:00100000:3.0:1767320516.827596:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985444992 00000100:00100000:3.0:1767320516.827604:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985444992 00000100:00080000:2.0:1767320516.827613:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.827616:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320516.827618:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320516.827624:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320516.827625:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5739 00000100:00080000:2.0:1767320516.827630:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.827633:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320516.827638:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985445248 00000100:00100000:3.0:1767320516.827643:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985445248 00000100:00100000:2.0:1767320516.827644:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939f80 x1853163985446016/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320516.827654:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985445376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320516.827677:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320516.827686:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985445376 00000100:00100000:3.0:1767320516.827688:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985445376 00000100:00100000:2.0:1767320516.827699:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5268 00000100:00100000:2.0:1767320516.827711:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985445248:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320516.827749:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985445248:12345-0@lo:400:kworker.0 Request processed in 42us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320516.827998:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985445632:0@lo:38:kworker.0 00000100:00100000:1.0:1767320516.828021:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320516.828105:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985446016:0@lo:250:kworker.0 00000400:00000100:1.0:1767320516.828124:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985446016 offset 0 length 520: 4 00000100:00100000:1.0:1767320516.828152:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985445760:0@lo:400:kworker.0 00000100:00100000:1.0:1767320516.828195:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320516.828378:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985445888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320516.828491:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320516.828500:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985445504:0@lo:400:kworker.0 00000100:00100000:1.0:1767320516.828527:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320516.828552:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985445120 00000100:00100000:1.0:1767320516.828556:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985445120 00000100:00100000:1.0:1767320516.828581:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1981 00000100:00100000:1.0:1767320516.828586:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985445120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320516.828603:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320516.828636:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985445120:12345-0@lo:38:kworker.0 Request processed in 48us (1014us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320516.828642:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1981 00000100:00100000:1.0:1767320516.828663:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985445632 00000100:00100000:1.0:1767320516.828666:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985445632 00000100:00100000:3.0:1767320516.828747:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985445504 00000100:00100000:3.0:1767320516.828751:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985445504 00000100:00100000:3.0:1767320516.828773:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5269 00000100:00100000:3.0:1767320516.828779:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985445504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320516.828829:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985445504:12345-0@lo:400:kworker.0 Request processed in 50us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320516.828835:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5269 00000100:00100000:3.0:1767320516.828849:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5962 00000100:00100000:3.0:1767320516.828853:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985444992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320516.828863:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320516.828887:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985444992:12345-0@lo:38:kworker.0 Request processed in 32us (1419us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320516.828893:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5962 00000100:00100000:3.0:1767320516.828904:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5963 00000100:00100000:3.0:1767320516.828908:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985445376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320516.828917:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320516.828937:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985445376:12345-0@lo:38:kworker.0 Request processed in 27us (1259us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320516.828941:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5963 00000100:00100000:3.0:1767320516.828964:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985445760 00000100:00100000:3.0:1767320516.828967:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985445760 00000100:00100000:3.0:1767320516.828990:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3425 00000100:00100000:3.0:1767320516.828996:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985445760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320516.829023:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985445760:12345-0@lo:400:kworker.0 Request processed in 27us (827us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320516.829028:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3425 00000100:00080000:3.0:1767320516.829044:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320516.829048:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320516.829086:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985445248:0@lo:400:kworker.0 00000100:00080000:3.0:1767320516.829127:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320516.829131:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320516.829134:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985445120:0@lo:38:kworker.0 00000100:00100000:0.0:1767320516.829362:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320516.829413:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985445888 00000100:00100000:0.0:1767320516.829417:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985445888 00000100:00100000:0.0:1767320516.829447:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3426 00000100:00100000:0.0:1767320516.829453:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985445888:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320516.829486:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985445888:12345-0@lo:400:kworker.0 Request processed in 33us (124us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320516.829491:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3426 00000100:00080000:0.0:1767320516.829507:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320516.829522:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985445888:0@lo:400:kworker.0 00000100:00100000:0.0:1767320516.829626:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985445504:0@lo:400:kworker.0 00000100:00100000:0.0:1767320516.829636:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985445760:0@lo:400:kworker.0 00000100:00080000:3.0:1767320516.830274:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320516.830281:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320516.830285:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985445376:0@lo:38:kworker.0 00000100:00080000:3.0:1767320516.830302:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320516.830306:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320516.830309:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985444992:0@lo:38:kworker.0 00000100:00100000:2.0:1767320516.830438:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5268 00000100:00100000:1.0:1767320516.832939:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5964 00000100:00100000:1.0:1767320516.832950:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985445632:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320516.832973:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320516.833193:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985445632:12345-0@lo:38:kworker.0 Request processed in 242us (5170us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320516.833201:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5964 00000100:00080000:3.0:1767320516.841962:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320516.841975:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320516.841981:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985445632:0@lo:38:kworker.0 00010000:00010000:1.0:1767320517.467417:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320518.363512:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985446144:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320518.363571:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320518.363679:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985446144 00000100:00100000:2.0:1767320518.363687:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985446144 00000100:00100000:2.0:1767320518.363786:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2626 00000100:00100000:2.0:1767320518.363796:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985446144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320518.363830:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320518.363887:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985446144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (314us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320518.363894:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2626 00000100:00100000:0.0:1767320518.363977:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985446144:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320518.364148:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985446272:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320518.364196:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320518.364267:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985446272 00000100:00100000:0.0:1767320518.364272:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985446272 00000100:00080000:3.0:1767320518.364441:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320518.364446:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320518.364517:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2627 00000100:00100000:0.0:1767320518.364522:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985446272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320518.364535:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320518.364565:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985446272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320518.364570:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2627 00000100:00100000:1.0:1767320518.364640:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985446272:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320518.493868:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320519.515595:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320520.539366:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320521.564723:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320521.898506:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320521.898553:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320521.898634:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920621824 00000100:00100000:3.0:1767320521.898640:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920621696 00000100:00100000:0.0:1767320521.898645:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920621824 00000100:00100000:3.0:1767320521.898648:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920621696 00000100:00100000:3.0:1767320521.898695:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5270 00000100:00100000:0.0:1767320521.898736:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5965 00000100:00100000:3.0:1767320521.898738:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920621824:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320521.898746:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920621696:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320521.898768:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320521.898821:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920621824:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 116us (324us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320521.898830:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920621696:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 74us (264us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320521.898831:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5270 00000100:00100000:0.0:1767320521.898843:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5965 00000100:00080000:2.0:1767320521.899136:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320521.947509:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.947519:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320521.947522:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.947527:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.947531:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.947539:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.947544:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.947550:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320521.947588:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61180 x1853163985446400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320521.947615:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.947618:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.947620:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.947623:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.947626:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.947629:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320521.947640:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61500 x1853163985446528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320521.947677:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.947681:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.947682:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.947685:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.947688:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.947691:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320521.947700:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffea00 x1853163985446784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320521.947731:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.947735:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.947736:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.947739:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.947741:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.947745:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320521.947754:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10000 x1853163985447040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320521.947791:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985447296:0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.947849:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320521.947907:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320521.947913:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985446656:0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.947937:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320521.947969:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985447168:0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.947990:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320521.948007:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320521.948012:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985446912:0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.948028:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320521.948052:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447296 00000100:00100000:1.0:1767320521.948060:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447296 00000100:00100000:1.0:1767320521.948103:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447168 00000100:00100000:1.0:1767320521.948106:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447168 00000100:00100000:1.0:1767320521.948125:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3427 00000100:00100000:1.0:1767320521.948133:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985447296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.948198:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985447296:12345-0@lo:400:kworker.0 Request processed in 65us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320521.948205:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3427 00000100:00100000:1.0:1767320521.948219:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3428 00000100:00100000:1.0:1767320521.948225:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985447168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.948257:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985447168:12345-0@lo:400:kworker.0 Request processed in 31us (265us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320521.948263:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3428 00000100:00100000:1.0:1767320521.948289:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985446656 00000100:00100000:1.0:1767320521.948292:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985446656 00000100:00100000:1.0:1767320521.948314:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985446912 00000100:00100000:1.0:1767320521.948316:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985446912 00000100:00080000:2.0:1767320521.948522:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320521.948528:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320521.948530:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320521.948608:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5271 00000100:00100000:1.0:1767320521.948619:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985446656:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.948655:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985446656:12345-0@lo:400:kworker.0 Request processed in 39us (717us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320521.953196:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5272 00000100:00100000:2.0:1767320521.953228:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985446912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320521.953337:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985446912:12345-0@lo:400:kworker.0 Request processed in 127us (5307us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320521.953347:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5272 00000100:00100000:2.0:1767320521.953449:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985446656:0@lo:400:kworker.0 00000100:00100000:2.0:1767320521.953465:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985447296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320521.953524:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985446400:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.953560:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320521.953593:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985446528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.953616:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320521.953644:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985446784:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.953663:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320521.953677:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985447040:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.953694:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320521.953735:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985446400 00000100:00100000:2.0:1767320521.953741:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985446400 00000100:00100000:2.0:1767320521.953760:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985446784 00000100:00100000:2.0:1767320521.953762:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985446784 00000100:00100000:2.0:1767320521.953769:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447040 00000100:00100000:2.0:1767320521.953771:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447040 00000100:00100000:2.0:1767320521.953781:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5966 00000100:00100000:2.0:1767320521.953786:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985446400:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320521.953806:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.953835:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985446400:12345-0@lo:38:kworker.0 Request processed in 48us (274us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320521.953842:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5966 00000100:00100000:2.0:1767320521.953854:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5967 00000100:00100000:2.0:1767320521.953857:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985446784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320521.953866:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.953888:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985446784:12345-0@lo:38:kworker.0 Request processed in 28us (223us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320521.953893:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5967 00000100:00100000:2.0:1767320521.953904:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5968 00000100:00100000:2.0:1767320521.953908:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985447040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320521.953916:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.953940:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985447040:12345-0@lo:38:kworker.0 Request processed in 30us (244us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320521.953946:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5968 00000100:00080000:2.0:1767320521.953970:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320521.953974:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320521.953977:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985446400:0@lo:38:kworker.0 00000100:00080000:2.0:1767320521.953984:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320521.953987:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320521.953990:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985446784:0@lo:38:kworker.0 00000100:00080000:2.0:1767320521.953995:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320521.953998:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320521.954001:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985447040:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.954035:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985446528 00000100:00100000:2.0:1767320521.954038:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985446528 00000100:00100000:2.0:1767320521.954066:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1982 00000100:00100000:2.0:1767320521.954071:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985446528:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320521.954080:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.954109:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985446528:12345-0@lo:38:kworker.0 Request processed in 37us (492us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320521.954114:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1982 00000100:00080000:2.0:1767320521.954136:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320521.954140:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320521.954143:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985446528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.954213:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985446912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320521.954224:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985447168:0@lo:400:kworker.0 00000100:00100000:1.0:1767320521.958319:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5271 00010000:00010000:2.0:1767320522.587481:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320523.483542:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985447424:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320523.483615:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320523.483783:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447424 00000100:00100000:2.0:1767320523.483792:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447424 00000100:00100000:2.0:1767320523.483864:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2628 00000100:00100000:2.0:1767320523.483874:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985447424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320523.483905:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320523.483992:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985447424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320523.484076:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320523.484087:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985447552:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320523.484494:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320523.488554:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985447424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4672us (4929us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320523.488571:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2628 00000100:00100000:1.0:1767320523.489360:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320523.489409:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447552 00000100:00100000:3.0:1767320523.489421:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447552 00000100:00100000:3.0:1767320523.489540:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2629 00000100:00100000:3.0:1767320523.489550:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985447552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320523.489592:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320523.489667:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985447552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 119us (307us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320523.489685:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2629 00000100:00100000:1.0:1767320523.489950:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985447552:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320523.490260:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320523.611401:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320524.635418:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320525.659405:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320526.683395:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320527.019550:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320527.019712:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320527.028416:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920622080 00000100:00100000:0.0:1767320527.028433:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920622080 00000100:00100000:0.0:1767320527.028535:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920621952 00000100:00100000:0.0:1767320527.028539:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920621952 00000100:00100000:0.0:1767320527.028562:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5273 00000100:00100000:0.0:1767320527.028573:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920622080:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320527.028672:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920622080:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 102us (9132us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320527.028686:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5273 00000100:00100000:0.0:1767320527.028844:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5969 00000100:00100000:0.0:1767320527.028849:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920621952:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320527.028869:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320527.028905:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920621952:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (9188us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320527.028912:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5969 00000100:00080000:3.0:1767320527.029529:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320527.068687:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320527.068694:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320527.068697:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320527.068701:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320527.068706:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320527.068713:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5770 00000100:00080000:1.0:1767320527.068718:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320527.068726:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320527.068760:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10a80 x1853163985447680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320527.068776:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320527.068779:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320527.068781:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320527.068784:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5770 00000100:00080000:1.0:1767320527.068787:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320527.068790:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320527.068849:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13b80 x1853163985447808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320527.068889:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320527.068893:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320527.068895:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320527.068899:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5770 00000100:00080000:1.0:1767320527.068901:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320527.068904:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320527.068916:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3480 x1853163985448064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320527.068942:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320527.068945:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320527.068946:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320527.068949:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5770 00000100:00080000:1.0:1767320527.068952:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320527.068954:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320527.068964:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985448320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320527.069135:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985448576:0@lo:400:kworker.0 00000100:00100000:3.0:1767320527.069225:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985447936:0@lo:400:kworker.0 00000100:00100000:3.0:1767320527.069279:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320527.069550:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985447680:0@lo:38:kworker.0 00000100:00100000:0.0:1767320527.069585:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985448448:0@lo:400:kworker.0 00000100:00100000:0.0:1767320527.069623:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320527.076114:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985448192:0@lo:400:kworker.0 00000100:00100000:2.0:1767320527.083981:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320527.084064:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447936 00000100:00100000:2.0:1767320527.084073:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447936 00000100:00100000:2.0:1767320527.084126:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448192 00000100:00100000:2.0:1767320527.084130:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448192 00000100:00100000:2.0:1767320527.084155:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5274 00000100:00100000:2.0:1767320527.084197:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985447936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320527.084265:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985447936:12345-0@lo:400:kworker.0 Request processed in 102us (14986us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320527.084273:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5274 00000100:00100000:2.0:1767320527.084290:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5275 00000100:00100000:2.0:1767320527.084296:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985448192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320527.084326:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985448192:12345-0@lo:400:kworker.0 Request processed in 30us (353us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320527.084332:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5275 00000100:00100000:2.0:1767320527.084374:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985448192:0@lo:400:kworker.0 00000100:00080000:2.0:1767320527.084396:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320527.084436:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985447936:0@lo:400:kworker.0 00000100:00100000:3.0:1767320527.086443:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320527.086501:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985447808:0@lo:38:kworker.0 00000100:00100000:3.0:1767320527.086555:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320527.086576:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985448064:0@lo:38:kworker.0 00000100:00100000:3.0:1767320527.086599:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320527.086626:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985448320:0@lo:38:kworker.0 00000100:00100000:3.0:1767320527.086649:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320527.086688:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447808 00000100:00100000:3.0:1767320527.086695:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447808 00000100:00100000:3.0:1767320527.086729:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1983 00000100:00100000:3.0:1767320527.086735:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985447808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320527.086755:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320527.086792:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985447808:12345-0@lo:38:kworker.0 Request processed in 56us (236us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320527.086799:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1983 00000100:00100000:3.0:1767320527.086822:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985447680 00000100:00100000:3.0:1767320527.086826:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985447680 00000100:00100000:3.0:1767320527.086843:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448064 00000100:00100000:3.0:1767320527.086845:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448064 00000100:00100000:3.0:1767320527.086858:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448320 00000100:00100000:3.0:1767320527.086861:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448320 00000100:00100000:3.0:1767320527.086871:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5970 00000100:00100000:3.0:1767320527.086875:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985447680:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320527.086885:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320527.088132:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5971 00000100:00100000:0.0:1767320527.088140:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985448064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320527.088155:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320527.088379:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985448064:12345-0@lo:38:kworker.0 Request processed in 238us (1779us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320527.088386:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5971 00000100:00100000:0.0:1767320527.088403:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5972 00000100:00100000:0.0:1767320527.088407:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985448320:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320527.088416:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320527.088436:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985448320:12345-0@lo:38:kworker.0 Request processed in 28us (1786us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320527.088442:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5972 00000100:00100000:0.0:1767320527.088477:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448448 00000100:00100000:0.0:1767320527.088483:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448448 00000100:00100000:0.0:1767320527.088530:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3429 00000100:00100000:0.0:1767320527.088538:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985448448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320527.088577:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985448448:12345-0@lo:400:kworker.0 Request processed in 41us (18953us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320527.088583:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3429 00000100:00080000:0.0:1767320527.088599:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320527.088636:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985448448:0@lo:400:kworker.0 00000100:00100000:0.0:1767320527.088669:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320527.088692:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448576 00000100:00100000:0.0:1767320527.088695:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448576 00000100:00100000:0.0:1767320527.088721:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3430 00000100:00100000:0.0:1767320527.088727:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985448576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320527.088755:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985448576:12345-0@lo:400:kworker.0 Request processed in 28us (84us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320527.088761:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3430 00000100:00080000:0.0:1767320527.088774:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320527.088791:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985448576:0@lo:400:kworker.0 00000100:00100000:3.0:1767320527.089154:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985447680:12345-0@lo:38:kworker.0 Request processed in 2276us (2717us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320527.089635:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5970 00000100:00080000:3.0:1767320527.089680:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320527.089686:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320527.089689:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985447680:0@lo:38:kworker.0 00000100:00080000:3.0:1767320527.089701:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320527.089704:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320527.089706:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985447808:0@lo:38:kworker.0 00000100:00080000:3.0:1767320527.089715:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320527.089718:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320527.089720:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985448064:0@lo:38:kworker.0 00000100:00080000:3.0:1767320527.089729:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320527.089732:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320527.089734:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985448320:0@lo:38:kworker.0 00010000:00010000:2.0:1767320527.707410:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320528.603513:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985448704:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320528.603682:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320528.603823:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448704 00000100:00100000:0.0:1767320528.603831:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448704 00000100:00100000:0.0:1767320528.603882:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2630 00000100:00100000:0.0:1767320528.603892:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985448704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320528.603921:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320528.604054:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985448832:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320528.604076:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320528.604098:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448832 00000100:00100000:3.0:1767320528.604101:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448832 00000100:00100000:3.0:1767320528.604118:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2631 00000100:00100000:3.0:1767320528.604123:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985448832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320528.604135:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320528.604154:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985448832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 31us (77us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320528.604158:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2631 00000100:00100000:3.0:1767320528.604226:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985448832:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320528.613795:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985448704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 9892us (10100us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320528.613821:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2630 00000100:00100000:2.0:1767320528.616968:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985448704:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320528.739348:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320529.756436:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320530.779401:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320531.805131:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320532.140324:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320532.140407:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320532.140572:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920622336 00000100:00100000:2.0:1767320532.140582:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920622336 00000100:00100000:2.0:1767320532.140665:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920622208 00000100:00100000:2.0:1767320532.140669:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920622208 00000100:00100000:2.0:1767320532.140686:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5276 00000100:00100000:2.0:1767320532.140696:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920622336:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320532.140769:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920622336:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (455us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320532.140776:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5276 00000100:00100000:2.0:1767320532.140793:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5973 00000100:00100000:2.0:1767320532.140797:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920622208:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320532.140822:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320532.140846:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920622208:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (440us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320532.140852:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5973 00000100:00080000:1.0:1767320532.141131:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320532.187272:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320532.187282:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320532.187285:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320532.187288:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320532.187292:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320532.187300:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5775 00000100:00080000:1.0:1767320532.187305:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320532.187310:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320532.187350:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac380 x1853163985448960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320532.187375:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320532.187378:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320532.187380:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320532.187384:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5775 00000100:00080000:1.0:1767320532.187386:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320532.187389:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320532.187401:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac000 x1853163985449088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320532.187442:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320532.187445:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320532.187447:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320532.187451:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5775 00000100:00080000:1.0:1767320532.187453:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320532.187456:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320532.187470:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f480 x1853163985449344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320532.187494:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320532.187497:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320532.187499:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320532.187502:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5775 00000100:00080000:1.0:1767320532.187505:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320532.187508:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320532.187546:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985449600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320532.187551:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985449216:0@lo:400:kworker.0 00000100:00100000:1.0:1767320532.187580:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985449472:0@lo:400:kworker.0 00000100:00100000:2.0:1767320532.187675:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985449856:0@lo:400:kworker.0 00000100:00100000:0.0:1767320532.187714:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320532.187751:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985448960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320532.187752:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449216 00000100:00100000:0.0:1767320532.187759:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449216 00000100:00100000:3.0:1767320532.187778:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320532.187805:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985448960 00000100:00100000:0.0:1767320532.187808:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985448960 00000100:00100000:3.0:1767320532.187826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985449088:0@lo:38:kworker.0 00000100:00100000:3.0:1767320532.187857:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320532.187879:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985449344:0@lo:38:kworker.0 00000100:00100000:3.0:1767320532.187900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320532.187906:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5277 00000100:00100000:0.0:1767320532.187916:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985449216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320532.187916:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985449600:0@lo:38:kworker.0 00000100:00100000:3.0:1767320532.187935:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320532.187963:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449088 00000100:00100000:3.0:1767320532.187967:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449088 00000100:00100000:0.0:1767320532.187972:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985449216:12345-0@lo:400:kworker.0 Request processed in 59us (259us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320532.187980:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5277 00000100:00100000:0.0:1767320532.187994:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449344 00000100:00100000:0.0:1767320532.187997:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449344 00000100:00100000:3.0:1767320532.187998:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1984 00000100:00100000:3.0:1767320532.188002:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985449088:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320532.188004:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449600 00000100:00100000:0.0:1767320532.188006:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449600 00000100:00100000:0.0:1767320532.188014:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5974 00000100:00100000:0.0:1767320532.188017:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985448960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320532.188019:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767320532.188024:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320532.188065:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320532.188082:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320532.188087:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985449728:0@lo:400:kworker.0 00000100:00100000:1.0:1767320532.188112:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320532.188115:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985448960:12345-0@lo:38:kworker.0 Request processed in 97us (336us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320532.188122:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5974 00000100:00100000:0.0:1767320532.188131:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449472 00000100:00100000:0.0:1767320532.188134:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449472 00000100:00100000:1.0:1767320532.188135:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449728 00000100:00100000:1.0:1767320532.188138:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449728 00000100:00100000:0.0:1767320532.188150:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5278 00000100:00100000:0.0:1767320532.188155:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985449472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320532.188161:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3431 00000100:00100000:1.0:1767320532.188184:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985449728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320532.188204:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985449472:12345-0@lo:400:kworker.0 Request processed in 49us (139us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320532.188208:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985449728:12345-0@lo:400:kworker.0 Request processed in 26us (96us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320532.188210:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5278 00000100:00100000:1.0:1767320532.188214:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3431 00000100:00100000:0.0:1767320532.188223:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5975 00000100:00100000:0.0:1767320532.188226:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985449344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320532.188235:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320532.188244:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985449728:0@lo:400:kworker.0 00000100:00100000:1.0:1767320532.188254:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985449472:0@lo:400:kworker.0 00000100:00100000:0.0:1767320532.188255:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985449344:12345-0@lo:38:kworker.0 Request processed in 27us (354us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320532.188260:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5975 00000100:00100000:0.0:1767320532.188269:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5976 00000100:00100000:0.0:1767320532.188273:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985449600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320532.188281:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320532.188297:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320532.188300:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985449600:12345-0@lo:38:kworker.0 Request processed in 26us (365us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320532.188301:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320532.188305:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5976 00000100:00100000:1.0:1767320532.188315:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985449216:0@lo:400:kworker.0 00000100:00100000:3.0:1767320532.188455:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985449088:12345-0@lo:38:kworker.0 Request processed in 450us (596us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320532.188461:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1984 00000100:00080000:3.0:1767320532.188487:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320532.188492:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320532.188496:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985448960:0@lo:38:kworker.0 00000100:00080000:3.0:1767320532.188505:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320532.188507:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320532.188510:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985449088:0@lo:38:kworker.0 00000100:00080000:3.0:1767320532.188526:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320532.188530:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320532.188532:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985449344:0@lo:38:kworker.0 00000100:00080000:3.0:1767320532.188539:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320532.188542:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320532.188544:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985449600:0@lo:38:kworker.0 00000100:00100000:2.0:1767320532.200748:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320532.200990:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449856 00000100:00100000:3.0:1767320532.200998:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449856 00000100:00100000:3.0:1767320532.201073:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3432 00000100:00100000:3.0:1767320532.201083:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985449856:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320532.201140:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985449856:12345-0@lo:400:kworker.0 Request processed in 60us (400us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320532.201148:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3432 00000100:00080000:1.0:1767320532.201345:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320532.202108:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985449856:0@lo:400:kworker.0 00010000:00010000:1.0:1767320532.830945:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320533.724058:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985449984:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320533.724127:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320533.724265:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985449984 00000100:00100000:0.0:1767320533.724274:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985449984 00000100:00100000:0.0:1767320533.724326:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2632 00000100:00100000:0.0:1767320533.724335:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985449984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320533.724371:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320533.724416:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985449984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320533.724423:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2632 00000100:00100000:1.0:1767320533.724517:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985449984:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320533.731273:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985450112:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320533.731337:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320533.731449:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985450112 00000100:00100000:3.0:1767320533.731457:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985450112 00000100:00100000:3.0:1767320533.731522:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2633 00000100:00100000:3.0:1767320533.731532:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985450112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320533.731558:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320533.731610:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985450112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320533.731622:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2633 00000100:00100000:2.0:1767320533.731701:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985450112:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320533.851383:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320534.875478:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320535.899864:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320536.929258:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320537.258042:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320537.258090:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320537.258228:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920622464 00000100:00100000:1.0:1767320537.258239:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920622464 00000100:00100000:1.0:1767320537.258281:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920622592 00000100:00100000:1.0:1767320537.258285:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920622592 00000100:00100000:1.0:1767320537.258341:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5279 00000100:00100000:1.0:1767320537.258351:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920622592:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320537.258432:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920622592:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 73us (332us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320537.258439:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5279 00000100:00100000:1.0:1767320537.258457:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5977 00000100:00100000:1.0:1767320537.258472:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920622464:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320537.258492:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320537.258528:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920622464:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (485us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320537.258535:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5977 00000100:00080000:3.0:1767320537.259221:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320537.307368:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320537.307379:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320537.307382:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320537.307386:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320537.307390:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320537.307399:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5780 00000100:00080000:0.0:1767320537.307403:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320537.307410:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320537.307459:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1500 x1853163985450240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320537.307571:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320537.307575:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320537.307577:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320537.307581:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5780 00000100:00080000:0.0:1767320537.307583:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320537.307586:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320537.307601:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3800 x1853163985450368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320537.307642:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320537.307646:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320537.307648:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320537.307651:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5780 00000100:00080000:0.0:1767320537.307653:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320537.307656:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320537.307668:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3100 x1853163985450624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320537.307697:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320537.307700:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320537.307702:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320537.307705:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5780 00000100:00080000:0.0:1767320537.307708:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320537.307711:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320537.307726:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9b100 x1853163985450880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320537.307855:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985450752:0@lo:400:kworker.0 00000100:00100000:0.0:1767320537.307915:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320537.308068:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985451136:0@lo:400:kworker.0 00000100:00100000:2.0:1767320537.308126:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985450496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320537.308192:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985450240:0@lo:38:kworker.0 00000100:00100000:0.0:1767320537.308268:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320537.308274:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985451008:0@lo:400:kworker.0 00000100:00100000:1.0:1767320537.311982:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320537.312089:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985450752 00000100:00100000:1.0:1767320537.312098:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985450752 00000100:00100000:1.0:1767320537.312158:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5280 00000100:00100000:1.0:1767320537.312182:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985450752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320537.312234:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985450752:12345-0@lo:400:kworker.0 Request processed in 68us (4318us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320537.312241:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5280 00000100:00100000:1.0:1767320537.312275:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985451136 00000100:00100000:1.0:1767320537.312279:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985451136 00000100:00100000:1.0:1767320537.312304:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3433 00000100:00100000:1.0:1767320537.312309:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a00c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985451136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320537.312341:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a00c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985451136:12345-0@lo:400:kworker.0 Request processed in 31us (367us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320537.312346:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3433 00000100:00100000:1.0:1767320537.312599:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985451136:0@lo:400:kworker.0 00000100:00080000:1.0:1767320537.312626:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320537.312630:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320537.313026:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320537.313067:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985450496 00000100:00100000:2.0:1767320537.313071:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985450496 00000100:00100000:2.0:1767320537.313102:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5281 00000100:00100000:2.0:1767320537.313110:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985450496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320537.313146:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985450496:12345-0@lo:400:kworker.0 Request processed in 38us (121us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320537.313152:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5281 00000100:00080000:2.0:1767320537.313181:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320537.313199:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985450496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320537.313491:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320537.313513:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985450368:0@lo:38:kworker.0 00000100:00100000:3.0:1767320537.313541:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320537.313570:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985450624:0@lo:38:kworker.0 00000100:00100000:3.0:1767320537.313591:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320537.313605:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985450880:0@lo:38:kworker.0 00000100:00100000:3.0:1767320537.313623:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320537.313645:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985450240 00000100:00100000:3.0:1767320537.313649:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985450240 00000100:00100000:3.0:1767320537.313668:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985450624 00000100:00100000:3.0:1767320537.313671:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985450624 00000100:00100000:3.0:1767320537.313683:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985450880 00000100:00100000:3.0:1767320537.313686:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985450880 00000100:00100000:3.0:1767320537.313699:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5978 00000100:00100000:3.0:1767320537.313703:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985450240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320537.313719:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320537.313760:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985450240:12345-0@lo:38:kworker.0 Request processed in 55us (269us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320537.313766:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5978 00000100:00100000:3.0:1767320537.313781:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5979 00000100:00100000:3.0:1767320537.313784:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985450624:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320537.313793:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320537.313815:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985450624:12345-0@lo:38:kworker.0 Request processed in 30us (224us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320537.313821:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5979 00000100:00100000:3.0:1767320537.313831:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5980 00000100:00100000:3.0:1767320537.313835:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985450880:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320537.313843:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320537.313863:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985450880:12345-0@lo:38:kworker.0 Request processed in 27us (240us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320537.313869:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5980 00000100:00080000:3.0:1767320537.313901:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320537.313906:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320537.313910:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985450880:0@lo:38:kworker.0 00000100:00080000:3.0:1767320537.313927:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320537.313930:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320537.313933:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985450240:0@lo:38:kworker.0 00000100:00080000:3.0:1767320537.314196:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320537.314200:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320537.314203:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985450624:0@lo:38:kworker.0 00000100:00100000:2.0:1767320537.315856:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985450368 00000100:00100000:2.0:1767320537.315867:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985450368 00000100:00100000:2.0:1767320537.315918:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1985 00000100:00100000:2.0:1767320537.315925:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985450368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320537.315952:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320537.316014:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985450368:12345-0@lo:38:kworker.0 Request processed in 88us (2472us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320537.316021:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1985 00000100:00080000:3.0:1767320537.316262:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320537.316268:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320537.316271:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985450368:0@lo:38:kworker.0 00000100:00100000:0.0:1767320537.321252:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320537.321326:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985450752:0@lo:400:kworker.0 00000100:00100000:1.0:1767320537.321454:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985451008 00000100:00100000:1.0:1767320537.321471:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985451008 00000100:00100000:1.0:1767320537.321542:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3434 00000100:00100000:1.0:1767320537.321552:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985451008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320537.321614:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985451008:12345-0@lo:400:kworker.0 Request processed in 63us (368us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320537.321621:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3434 00000100:00100000:0.0:1767320537.321877:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985451008:0@lo:400:kworker.0 00000100:00080000:2.0:1767320537.322086:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320537.947877:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320538.844156:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320538.844199:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985451264:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320538.844269:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320538.844478:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985451264 00000100:00100000:1.0:1767320538.844488:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985451264 00000100:00100000:1.0:1767320538.844545:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2634 00000100:00100000:1.0:1767320538.844554:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985451264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320538.844584:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320538.844636:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985451264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (366us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320538.844644:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2634 00000100:00100000:0.0:1767320538.844734:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985451264:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320538.845826:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985451392:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320538.845869:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320538.846120:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985451392 00000100:00100000:3.0:1767320538.846127:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985451392 00000100:00100000:3.0:1767320538.846201:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2635 00000100:00100000:3.0:1767320538.846210:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985451392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320538.846231:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320538.846276:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985451392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (406us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320538.846283:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2635 00000100:00100000:2.0:1767320538.846504:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985451392:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320538.972753:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320539.995788:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320541.019687:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320541.369222:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767320541.369258:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920622720 offset 0 length 520: 4 00000100:00100000:0.0:1767320541.369282:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320541.369554:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920622976 00000100:00100000:1.0:1767320541.369565:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920622976 00000100:00100000:1.0:1767320541.369644:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920622848 00000100:00100000:1.0:1767320541.369649:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920622848 00000100:00100000:1.0:1767320541.369669:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5282 00000100:00100000:1.0:1767320541.369679:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920622976:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320541.369749:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920622976:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (535us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320541.369757:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5282 00000100:00100000:3.0:1767320541.369779:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5981 00000100:00100000:3.0:1767320541.369788:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920622848:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320541.369807:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320541.369838:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920622848:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (555us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767320541.369919:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320541.379438:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5981 00000100:00100000:3.0:1767320541.403310:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320516/real 1767320516] req@ffff8ce1a5939f80 x1853163985446016/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320541 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320541.403353:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a5939f80 x1853163985446016/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320541 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320541.403373:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320541.415460:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320541.415470:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320541.415474:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320541.415482:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5765 00000100:00080000:0.0:1767320541.415487:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320541.415504:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320541.415536:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89180 x1853163985451520/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320541.415552:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320541.415555:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320541.415557:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320541.415560:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5785 00000100:00080000:0.0:1767320541.415563:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320541.415566:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320541.415579:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985451648/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320541.415587:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320541.415590:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320541.415591:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320541.415594:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5785 00000100:00080000:0.0:1767320541.415597:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320541.415600:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320541.415610:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8a680 x1853163985451776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320541.415658:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320541.415662:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320541.415663:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320541.415667:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5785 00000100:00080000:0.0:1767320541.415669:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320541.415672:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320541.415683:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88000 x1853163985452032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320541.415706:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320541.415709:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320541.415711:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320541.415714:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5785 00000100:00080000:0.0:1767320541.415716:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320541.415718:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320541.415728:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b480 x1853163985452288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320541.415768:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985452160:0@lo:400:kworker.0 00000100:00100000:0.0:1767320541.415839:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320541.415900:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320541.415906:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985452416:0@lo:400:kworker.0 00000100:00100000:0.0:1767320541.415931:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320541.415974:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985452544:0@lo:400:kworker.0 00000100:00100000:0.0:1767320541.415997:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320541.416012:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320541.416017:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985451904:0@lo:400:kworker.0 00000100:00100000:0.0:1767320541.416034:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320541.416075:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452416 00000100:00100000:0.0:1767320541.416082:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452416 00000100:00100000:0.0:1767320541.416126:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452544 00000100:00100000:0.0:1767320541.416129:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452544 00000100:00100000:0.0:1767320541.416148:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3435 00000100:00100000:0.0:1767320541.416156:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985452416:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.416531:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452160 00000100:00100000:2.0:1767320541.416536:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452160 00000100:00100000:2.0:1767320541.416576:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985451904 00000100:00100000:2.0:1767320541.416579:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985451904 00000100:00100000:2.0:1767320541.416600:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5283 00000100:00100000:2.0:1767320541.416606:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985452160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.416648:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985452160:12345-0@lo:400:kworker.0 Request processed in 42us (809us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320541.416655:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5283 00000100:00100000:2.0:1767320541.416671:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5284 00000100:00100000:2.0:1767320541.416677:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985451904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.416706:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985451904:12345-0@lo:400:kworker.0 Request processed in 29us (670us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320541.416711:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5284 00000100:00100000:2.0:1767320541.416817:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3436 00000100:00100000:2.0:1767320541.416825:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985452544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.416854:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985452544:12345-0@lo:400:kworker.0 Request processed in 31us (856us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320541.416860:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3436 00000100:00080000:2.0:1767320541.416876:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320541.416880:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320541.416882:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320541.416915:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985452160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320541.417071:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985451904:0@lo:400:kworker.0 00000100:00100000:1.0:1767320541.417400:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985452544:0@lo:400:kworker.0 00000100:00080000:3.0:1767320541.417557:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320541.417563:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985446016:0@lo:250:kworker.0 00000100:00100000:3.0:1767320541.417596:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985451520:0@lo:250:kworker.0 00000400:00000100:3.0:1767320541.417626:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985451520 offset 0 length 520: 4 00000100:00100000:3.0:1767320541.417671:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985451648:0@lo:38:kworker.0 00000100:00100000:3.0:1767320541.417696:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320541.417724:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985451776:0@lo:38:kworker.0 00000100:00100000:3.0:1767320541.417746:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320541.417769:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985451776 00000100:00100000:3.0:1767320541.417774:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985451776 00000100:00100000:3.0:1767320541.417798:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1986 00000100:00100000:3.0:1767320541.417803:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985451776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320541.417819:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320541.417846:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985451776:12345-0@lo:38:kworker.0 Request processed in 42us (99us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320541.417852:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1986 00000100:00080000:3.0:1767320541.417881:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320541.417885:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320541.417888:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985451776:0@lo:38:kworker.0 00000100:00100000:3.0:1767320541.417893:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985452032:0@lo:38:kworker.0 00000100:00100000:3.0:1767320541.417914:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320541.417928:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985452288:0@lo:38:kworker.0 00000100:00100000:3.0:1767320541.417945:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320541.417963:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985451648 00000100:00100000:3.0:1767320541.417966:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985451648 00000100:00100000:3.0:1767320541.417984:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452032 00000100:00100000:3.0:1767320541.417987:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452032 00000100:00100000:3.0:1767320541.417997:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452288 00000100:00100000:3.0:1767320541.417999:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452288 00000100:00100000:3.0:1767320541.418011:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5982 00000100:00100000:3.0:1767320541.418015:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985451648:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320541.418025:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320541.418048:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985451648:12345-0@lo:38:kworker.0 Request processed in 32us (351us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320541.418054:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5982 00000100:00100000:3.0:1767320541.418065:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5983 00000100:00100000:3.0:1767320541.418069:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985452032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320541.418077:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320541.418110:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985452032:12345-0@lo:38:kworker.0 Request processed in 39us (194us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320541.418115:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5983 00000100:00100000:3.0:1767320541.418124:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5984 00000100:00100000:3.0:1767320541.418127:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985452288:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320541.418134:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320541.418153:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985452288:12345-0@lo:38:kworker.0 Request processed in 24us (207us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320541.418159:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5984 00000100:00080000:3.0:1767320541.418245:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320541.418249:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320541.418252:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985452288:0@lo:38:kworker.0 00000100:00080000:3.0:1767320541.418268:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320541.418271:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320541.418275:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985451648:0@lo:38:kworker.0 00000100:00080000:3.0:1767320541.418282:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320541.418284:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320541.418287:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985452032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320541.422435:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985452416:12345-0@lo:400:kworker.0 Request processed in 6269us (6492us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320541.422449:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3435 00000100:00100000:0.0:1767320541.423932:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985452416:0@lo:400:kworker.0 00010000:00010000:1.0:1767320542.044716:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320543.067412:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320543.965907:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985452672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320543.965977:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320543.966090:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452672 00000100:00100000:2.0:1767320543.966099:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452672 00000100:00100000:2.0:1767320543.966191:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2636 00000100:00100000:2.0:1767320543.966202:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985452672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320543.966235:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320543.966279:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985452672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320543.966286:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2636 00000100:00100000:2.0:1767320543.966373:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985452672:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320543.967925:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320543.968202:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985452800:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320543.968257:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320543.968448:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452800 00000100:00100000:1.0:1767320543.968456:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452800 00000100:00100000:1.0:1767320543.968518:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2637 00000100:00100000:1.0:1767320543.968526:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985452800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320543.968552:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320543.968592:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985452800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (334us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320543.968599:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2637 00000100:00080000:3.0:1767320543.968663:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320543.968728:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985452800:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320544.091524:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320545.115749:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320546.139589:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320546.475115:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320546.475259:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320546.475366:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920623232 00000100:00100000:2.0:1767320546.475378:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920623232 00000100:00100000:2.0:1767320546.475457:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920623104 00000100:00100000:2.0:1767320546.475460:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920623104 00000100:00100000:2.0:1767320546.475478:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5285 00000100:00100000:2.0:1767320546.475488:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920623232:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320546.475574:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5985 00000100:00100000:2.0:1767320546.475574:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920623232:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (470us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320546.475580:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623104:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:2.0:1767320546.475581:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5285 00010000:00020000:0.0:1767320546.475600:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320546.476127:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320546.500048:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623104:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 24455us (24786us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320546.500062:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5985 00000100:00080000:1.0:1767320546.523356:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320546.523366:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320546.523369:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320546.523373:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320546.523377:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320546.523384:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5789 00000100:00080000:1.0:1767320546.523389:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320546.523397:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320546.523448:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985452928/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320546.523474:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320546.523478:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320546.523479:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320546.523483:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5789 00000100:00080000:1.0:1767320546.523485:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320546.523488:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320546.523500:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac700 x1853163985453056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320546.523536:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767320546.523571:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320546.523585:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985453184:0@lo:400:kworker.0 00000100:00100000:0.0:1767320546.523648:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320546.523710:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453184 00000100:00100000:0.0:1767320546.523717:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453184 00000100:00100000:0.0:1767320546.523768:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5286 00000100:00100000:0.0:1767320546.523776:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985453184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320546.523825:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985453184:12345-0@lo:400:kworker.0 Request processed in 49us (176us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320546.523827:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320546.523831:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5286 00000100:00100000:3.0:1767320546.523841:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985452928:0@lo:38:kworker.0 00000100:00100000:0.0:1767320546.523868:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985453184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320546.523868:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320546.523887:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985452928 00000100:00100000:3.0:1767320546.523889:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985453056:0@lo:38:kworker.0 00000100:00100000:0.0:1767320546.523890:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985452928 00000100:00100000:0.0:1767320546.523909:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5986 00000100:00100000:3.0:1767320546.523910:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320546.523914:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985452928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320546.523932:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320546.523940:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453056 00000100:00100000:3.0:1767320546.523944:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453056 00000100:00100000:0.0:1767320546.523961:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985452928:12345-0@lo:38:kworker.0 Request processed in 46us (91us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320546.523962:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1987 00000100:00100000:3.0:1767320546.523966:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985453056:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320546.523967:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5986 00010000:00020000:3.0:1767320546.523976:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320546.523999:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985453056:12345-0@lo:38:kworker.0 Request processed in 31us (89us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320546.524004:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1987 00000100:00080000:3.0:1767320546.524028:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320546.524033:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320546.524036:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985452928:0@lo:38:kworker.0 00000100:00080000:3.0:1767320546.524044:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320546.524047:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320546.524049:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985453056:0@lo:38:kworker.0 00000100:00080000:1.0:1767320546.524090:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320546.524092:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320546.524095:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5789 00000100:00080000:1.0:1767320546.524098:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320546.524101:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320546.524117:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac700 x1853163985453312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320546.524156:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320546.524160:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320546.524192:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320546.524196:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5789 00000100:00080000:1.0:1767320546.524199:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320546.524202:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320546.524214:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac000 x1853163985453568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320546.524262:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985453824:0@lo:400:kworker.0 00000100:00100000:1.0:1767320546.524291:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320546.524835:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985453312:0@lo:38:kworker.0 00000100:00100000:3.0:1767320546.524868:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320546.524886:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985453568:0@lo:38:kworker.0 00000100:00100000:3.0:1767320546.524904:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320546.524930:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453312 00000100:00100000:3.0:1767320546.524935:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453312 00000100:00100000:3.0:1767320546.524951:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453568 00000100:00100000:3.0:1767320546.524954:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453568 00000100:00100000:3.0:1767320546.524970:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5987 00000100:00100000:3.0:1767320546.524975:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985453312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320546.524989:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320546.525021:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985453312:12345-0@lo:38:kworker.0 Request processed in 45us (152us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320546.525028:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5987 00000100:00100000:3.0:1767320546.525041:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5988 00000100:00100000:3.0:1767320546.525045:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985453568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320546.525053:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320546.525073:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985453568:12345-0@lo:38:kworker.0 Request processed in 26us (167us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320546.525078:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5988 00000100:00080000:3.0:1767320546.525106:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320546.525110:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320546.525113:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985453312:0@lo:38:kworker.0 00000100:00080000:3.0:1767320546.525120:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320546.525124:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320546.525873:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985453568:0@lo:38:kworker.0 00000100:00100000:2.0:1767320546.526138:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985453696:0@lo:400:kworker.0 00000100:00100000:0.0:1767320546.526195:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985453440:0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.526213:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320546.526224:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320546.526264:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453440 00000100:00100000:2.0:1767320546.526264:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453824 00000100:00100000:0.0:1767320546.526269:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453440 00000100:00100000:2.0:1767320546.526269:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453824 00000100:00100000:0.0:1767320546.526308:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5287 00000100:00100000:0.0:1767320546.526316:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985453440:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320546.526316:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453696 00000100:00100000:1.0:1767320546.526320:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453696 00000100:00100000:0.0:1767320546.526357:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985453440:12345-0@lo:400:kworker.0 Request processed in 41us (131us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320546.526364:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5287 00000100:00100000:0.0:1767320546.526394:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985453440:0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.532013:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3437 00000100:00100000:2.0:1767320546.532031:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985453824:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.532130:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985453824:12345-0@lo:400:kworker.0 Request processed in 106us (7837us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320546.532138:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3437 00000100:00100000:2.0:1767320546.533791:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985453824:0@lo:400:kworker.0 00000100:00100000:1.0:1767320546.533954:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3438 00000100:00100000:1.0:1767320546.533967:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985453696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320546.534060:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985453696:12345-0@lo:400:kworker.0 Request processed in 96us (7845us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320546.534067:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3438 00000100:00080000:3.0:1767320546.534326:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320546.534332:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320546.534335:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320546.537680:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985453696:0@lo:400:kworker.0 00010000:00010000:2.0:1767320547.163760:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320548.191290:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320549.083440:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320549.083571:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320549.083591:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985454080:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320549.083632:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985453952:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320549.083660:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320549.083712:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320549.083797:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454080 00000100:00100000:1.0:1767320549.083797:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985453952 00000100:00100000:0.0:1767320549.083805:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454080 00000100:00100000:1.0:1767320549.083805:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985453952 00000100:00080000:3.0:1767320549.083915:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320549.083919:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320549.083955:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2638 00000100:00100000:1.0:1767320549.083966:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985453952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320549.084000:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320549.084052:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985453952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320549.084059:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2638 00000100:00100000:1.0:1767320549.084076:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2639 00000100:00100000:1.0:1767320549.084083:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985454080:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320549.084097:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320549.084128:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985454080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 46us (468us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320549.084134:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2639 00000100:00100000:3.0:1767320549.084223:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985453952:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320549.084268:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985454080:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320549.211580:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320550.235437:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320551.264077:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320551.594728:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320551.594784:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320551.594909:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920623488 00000100:00100000:1.0:1767320551.594919:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920623488 00000100:00100000:1.0:1767320551.594992:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920623360 00000100:00100000:1.0:1767320551.594996:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920623360 00000100:00100000:1.0:1767320551.595025:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5288 00000100:00100000:1.0:1767320551.595035:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920623488:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320551.595111:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920623488:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320551.595119:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5288 00000100:00100000:1.0:1767320551.595137:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5989 00000100:00100000:1.0:1767320551.595142:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623360:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320551.595219:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320551.595248:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623360:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 105us (464us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320551.595254:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5989 00000100:00080000:3.0:1767320551.595555:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320551.643281:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320551.643292:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320551.643295:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320551.643299:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320551.643303:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320551.643311:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5794 00000100:00080000:1.0:1767320551.643315:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320551.643322:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320551.643363:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ea00 x1853163985454208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320551.643397:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320551.643401:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320551.643402:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320551.643406:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5794 00000100:00080000:1.0:1767320551.643408:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320551.643412:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320551.643427:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612df80 x1853163985454336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320551.643475:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320551.643479:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320551.643482:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320551.643485:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5794 00000100:00080000:1.0:1767320551.643488:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320551.643491:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320551.643501:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bebb80 x1853163985454592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320551.643525:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320551.643528:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320551.643530:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320551.643533:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5794 00000100:00080000:1.0:1767320551.643535:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320551.643538:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320551.643547:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9b100 x1853163985454848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320551.643650:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985454464:0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.643718:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320551.643794:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320551.643799:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985455104:0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.643821:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320551.643843:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985454976:0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.643857:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320551.643868:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320551.643871:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985454720:0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.643881:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320551.643901:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985455104 00000100:00100000:2.0:1767320551.643908:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985455104 00000100:00100000:2.0:1767320551.643947:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454976 00000100:00100000:2.0:1767320551.643949:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454976 00000100:00100000:2.0:1767320551.643962:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3439 00000100:00100000:2.0:1767320551.643968:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985455104:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.644001:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985455104:12345-0@lo:400:kworker.0 Request processed in 34us (179us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320551.644015:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3439 00000100:00100000:2.0:1767320551.644032:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3440 00000100:00100000:2.0:1767320551.644038:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985454976:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.644070:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985454976:12345-0@lo:400:kworker.0 Request processed in 33us (212us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320551.644074:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3440 00000100:00100000:2.0:1767320551.644096:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454464 00000100:00100000:2.0:1767320551.644099:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454464 00000100:00100000:2.0:1767320551.644121:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454720 00000100:00100000:2.0:1767320551.644124:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454720 00000100:00100000:2.0:1767320551.644143:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5289 00000100:00100000:2.0:1767320551.644149:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985454464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.644246:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985454464:12345-0@lo:400:kworker.0 Request processed in 97us (527us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320551.644253:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5289 00000100:00100000:2.0:1767320551.644265:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5290 00000100:00100000:2.0:1767320551.644272:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985454720:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.644296:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985454720:12345-0@lo:400:kworker.0 Request processed in 26us (413us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320551.644301:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5290 00000100:00080000:2.0:1767320551.644319:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320551.644323:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320551.644325:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320551.644351:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985454720:0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.644361:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985454976:0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.644388:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985455104:0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.644396:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985454464:0@lo:400:kworker.0 00000100:00100000:3.0:1767320551.644504:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985454208:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.644535:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320551.644565:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985454336:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.644588:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320551.644611:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985454592:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.644637:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320551.644653:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985454848:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.644671:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320551.644704:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454336 00000100:00100000:3.0:1767320551.644708:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454336 00000100:00100000:3.0:1767320551.644731:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1988 00000100:00100000:3.0:1767320551.644735:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985454336:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320551.644749:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454208 00010000:00020000:3.0:1767320551.644752:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320551.644753:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454208 00000100:00100000:3.0:1767320551.644783:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985454336:12345-0@lo:38:kworker.0 Request processed in 46us (193us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320551.644790:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1988 00000100:00100000:3.0:1767320551.644812:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454592 00000100:00100000:3.0:1767320551.644815:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454592 00000100:00100000:3.0:1767320551.644832:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985454848 00000100:00100000:3.0:1767320551.644835:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985454848 00000100:00100000:3.0:1767320551.644844:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5990 00000100:00100000:3.0:1767320551.644848:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985454208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320551.644858:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320551.644882:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985454208:12345-0@lo:38:kworker.0 Request processed in 32us (346us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320551.644888:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5990 00000100:00100000:3.0:1767320551.644901:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5991 00000100:00100000:3.0:1767320551.644904:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985454592:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320551.644912:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320551.644933:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985454592:12345-0@lo:38:kworker.0 Request processed in 27us (295us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320551.644938:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5991 00000100:00100000:3.0:1767320551.644948:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5992 00000100:00100000:3.0:1767320551.644951:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985454848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320551.644961:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320551.644980:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985454848:12345-0@lo:38:kworker.0 Request processed in 28us (309us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320551.644985:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5992 00000100:00080000:3.0:1767320551.645011:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320551.645016:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320551.645019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985454208:0@lo:38:kworker.0 00000100:00080000:3.0:1767320551.645027:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320551.645030:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320551.645033:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985454336:0@lo:38:kworker.0 00000100:00080000:3.0:1767320551.645040:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320551.645043:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320551.645046:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985454592:0@lo:38:kworker.0 00000100:00080000:3.0:1767320551.645053:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320551.645056:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320551.645059:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985454848:0@lo:38:kworker.0 00010000:00010000:2.0:1767320552.283648:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320553.307418:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320554.203613:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985455232:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320554.203696:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320554.204190:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985455232 00000100:00100000:1.0:1767320554.204199:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985455232 00000100:00100000:1.0:1767320554.204268:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2640 00000100:00100000:1.0:1767320554.204277:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985455232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320554.204309:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320554.204358:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985455232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (662us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320554.204366:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2640 00000100:00100000:3.0:1767320554.204568:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985455360:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320554.204600:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320554.204762:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985455360 00000100:00100000:0.0:1767320554.204768:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985455360 00000100:00100000:0.0:1767320554.204805:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2641 00000100:00100000:0.0:1767320554.204813:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985455360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320554.204829:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320554.204862:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985455360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 50us (260us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320554.204871:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2641 00000100:00100000:3.0:1767320554.204950:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985455360:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320554.205150:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320554.205155:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320554.205201:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985455232:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320554.331396:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320555.355470:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320556.381306:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320556.714867:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320556.714943:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320556.714982:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920623616 00000100:00100000:1.0:1767320556.715000:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920623616 00000100:00100000:0.0:1767320556.715043:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920623744 00000100:00100000:0.0:1767320556.715060:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920623744 00000100:00100000:1.0:1767320556.715095:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5993 00000100:00100000:1.0:1767320556.715107:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623616:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320556.715138:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320556.715147:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5291 00000100:00100000:0.0:1767320556.715193:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920623744:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767320556.715376:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320556.715448:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623616:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 338us (583us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320556.715459:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5993 00000100:00100000:0.0:1767320556.715833:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920623744:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 672us (888us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320556.715842:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5291 00000100:00080000:0.0:1767320556.763360:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320556.763383:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320556.763386:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320556.763390:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320556.763394:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320556.763403:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5799 00000100:00080000:0.0:1767320556.763408:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320556.763414:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320556.763454:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594e00 x1853163985455488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320556.763484:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320556.763488:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320556.763490:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320556.763493:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5799 00000100:00080000:0.0:1767320556.763496:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320556.763499:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320556.763513:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594000 x1853163985455616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320556.763609:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985455744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320556.763682:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320556.763776:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320556.763781:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320556.763784:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320556.763791:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5799 00000100:00080000:0.0:1767320556.763795:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320556.763802:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320556.763802:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985455744 00000100:00100000:2.0:1767320556.763819:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985455744 00000100:00100000:0.0:1767320556.763821:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596300 x1853163985455872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320556.763869:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320556.763872:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320556.763874:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320556.763881:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5799 00000100:00080000:0.0:1767320556.763883:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320556.763889:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320556.763907:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596d80 x1853163985456128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320556.763923:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5292 00000100:00100000:2.0:1767320556.763940:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985455744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320556.764280:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985455488:0@lo:38:kworker.0 00000100:00100000:3.0:1767320556.764325:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320556.764384:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985455616:0@lo:38:kworker.0 00000100:00100000:3.0:1767320556.764414:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320556.764439:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985455872:0@lo:38:kworker.0 00000100:00100000:3.0:1767320556.764465:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320556.764486:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985456128:0@lo:38:kworker.0 00000100:00100000:3.0:1767320556.764514:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320556.764552:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985456384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320556.764579:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320556.764627:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985456000:0@lo:400:kworker.0 00000100:00100000:3.0:1767320556.764656:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320556.764684:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320556.764691:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985456256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320556.764716:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320556.764734:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985455488 00000100:00100000:0.0:1767320556.764746:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985455488 00000100:00100000:3.0:1767320556.764759:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985455872 00000100:00100000:3.0:1767320556.764766:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985455872 00000100:00100000:0.0:1767320556.764777:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456128 00000100:00100000:0.0:1767320556.764783:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456128 00000100:00100000:3.0:1767320556.764790:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456000 00000100:00100000:3.0:1767320556.764796:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456000 00000100:00100000:0.0:1767320556.764813:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5994 00000100:00100000:0.0:1767320556.764823:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985455488:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320556.764831:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5293 00000100:00100000:3.0:1767320556.764840:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985456000:12345-0@lo:400:kworker.0 00010000:00020000:0.0:1767320556.764851:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320556.764885:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985456000:12345-0@lo:400:kworker.0 Request processed in 44us (226us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320556.764895:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5293 00000100:00100000:3.0:1767320556.764919:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5995 00000100:00100000:3.0:1767320556.764923:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985455872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320556.764938:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320556.765092:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985455488:12345-0@lo:38:kworker.0 Request processed in 266us (763us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320556.765100:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5994 00000100:00100000:0.0:1767320556.765120:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5996 00000100:00100000:0.0:1767320556.765126:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985456128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320556.765137:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320556.766837:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985456128:12345-0@lo:38:kworker.0 Request processed in 1702us (2313us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320556.766848:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5996 00000100:00100000:0.0:1767320556.766894:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985455616 00000100:00100000:3.0:1767320556.766902:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985455872:12345-0@lo:38:kworker.0 Request processed in 1967us (2427us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320556.766904:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985455616 00000100:00100000:3.0:1767320556.766918:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5995 00000100:00100000:3.0:1767320556.766970:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456384 00000100:00100000:3.0:1767320556.766977:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456384 00000100:00100000:3.0:1767320556.767041:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456256 00000100:00100000:3.0:1767320556.767045:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456256 00000100:00100000:3.0:1767320556.767086:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3441 00000100:00100000:3.0:1767320556.767094:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985456384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320556.767218:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1989 00000100:00100000:3.0:1767320556.767222:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985456384:12345-0@lo:400:kworker.0 Request processed in 126us (2638us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320556.767233:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985455616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320556.767255:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320556.767304:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985455616:12345-0@lo:38:kworker.0 Request processed in 77us (2889us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320556.767310:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1989 00000100:00080000:0.0:1767320556.767340:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320556.767346:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320556.767349:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320556.767411:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3442 00000100:00100000:0.0:1767320556.767424:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985456256:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320556.767484:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985456256:12345-0@lo:400:kworker.0 Request processed in 66us (2768us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320556.767491:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3442 00000100:00100000:0.0:1767320556.767541:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985456256:0@lo:400:kworker.0 00000100:00100000:0.0:1767320556.767556:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985456000:0@lo:400:kworker.0 00000100:00080000:0.0:1767320556.767869:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320556.767875:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320556.767879:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985455488:0@lo:38:kworker.0 00000100:00080000:0.0:1767320556.767889:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320556.767892:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320556.767896:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985455616:0@lo:38:kworker.0 00000100:00080000:0.0:1767320556.767906:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320556.767909:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320556.767925:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985455872:0@lo:38:kworker.0 00000100:00080000:0.0:1767320556.767943:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320556.767946:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320556.767949:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985456128:0@lo:38:kworker.0 00000100:00100000:0.0:1767320556.768266:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985456384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320556.768580:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3441 00000100:00100000:2.0:1767320556.770213:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985455744:12345-0@lo:400:kworker.0 Request processed in 6263us (6518us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320556.770231:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5292 00000100:00100000:1.0:1767320556.770656:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985455744:0@lo:400:kworker.0 00010000:00010000:1.0:1767320557.405303:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320558.430990:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320559.323875:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320559.323905:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985456512:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320559.324013:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320559.324023:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985456640:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320559.324101:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456512 00000100:00100000:2.0:1767320559.324111:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456512 00000100:00100000:2.0:1767320559.324220:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2642 00000100:00100000:2.0:1767320559.324233:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985456512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320559.324274:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767320559.324372:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320559.327063:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320559.327338:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456640 00000100:00100000:1.0:1767320559.327353:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456640 00000100:00100000:1.0:1767320559.327457:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2643 00000100:00100000:1.0:1767320559.327470:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985456640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320559.327517:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320559.327584:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985456640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (534us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320559.327592:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2643 00000100:00080000:1.0:1767320559.327618:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320559.328089:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985456640:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320559.332359:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985456512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 8111us (8327us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320559.332384:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2642 00000100:00100000:3.0:1767320559.333509:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985456512:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320559.451535:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320560.478123:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320561.499690:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320561.841621:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320561.841684:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320561.841818:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920624000 00000100:00100000:0.0:1767320561.841833:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920624000 00000100:00100000:2.0:1767320561.841876:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920623872 00000100:00100000:2.0:1767320561.841881:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920623872 00000100:00080000:1.0:1767320561.841923:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320561.841936:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5294 00000100:00100000:0.0:1767320561.841943:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5997 00000100:00100000:0.0:1767320561.841951:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623872:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:2.0:1767320561.841951:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920624000:12345-192.168.204.26@tcp:400:kworker.0 00010000:00020000:0.0:1767320561.842006:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320561.842060:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920623872:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 109us (375us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320561.842069:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5997 00000100:00100000:2.0:1767320561.857247:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920624000:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 15281us (15619us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320561.857270:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5294 00000100:00080000:0.0:1767320561.883304:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320561.883316:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320561.883320:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320561.883324:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320561.883330:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320561.883340:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5805 00000100:00080000:0.0:1767320561.883346:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320561.883354:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320561.884260:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579180 x1853163985456768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320561.884283:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320561.884287:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320561.884289:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320561.884292:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5805 00000100:00080000:0.0:1767320561.884295:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320561.884300:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320561.884320:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a680 x1853163985456896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320561.884363:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320561.884367:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320561.884369:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320561.884374:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5805 00000100:00080000:0.0:1767320561.884377:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320561.884380:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320561.884657:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985457152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320561.884697:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320561.884700:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320561.884702:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320561.884706:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5805 00000100:00080000:0.0:1767320561.884709:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320561.884712:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320561.884722:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578e00 x1853163985457408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320561.884777:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985457536:0@lo:400:kworker.0 00000100:00100000:0.0:1767320561.884861:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320561.884891:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985457024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320561.884935:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320561.884953:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320561.884961:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985457280:0@lo:400:kworker.0 00000100:00100000:1.0:1767320561.884988:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320561.884998:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985457664:0@lo:400:kworker.0 00000100:00100000:0.0:1767320561.884999:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320561.885032:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320561.885086:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457536 00000100:00100000:1.0:1767320561.885095:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457536 00000100:00100000:1.0:1767320561.885161:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457664 00000100:00100000:1.0:1767320561.885191:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457664 00000100:00100000:1.0:1767320561.885221:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3443 00000100:00100000:1.0:1767320561.885232:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985457536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320561.885382:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985457536:12345-0@lo:400:kworker.0 Request processed in 151us (519us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320561.885392:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3443 00000100:00100000:1.0:1767320561.885409:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3444 00000100:00100000:1.0:1767320561.885414:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985457664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320561.885441:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985457664:12345-0@lo:400:kworker.0 Request processed in 28us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320561.885446:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3444 00000100:00080000:1.0:1767320561.885459:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320561.885462:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320561.885488:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985457664:0@lo:400:kworker.0 00000100:00100000:1.0:1767320561.885526:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985457536:0@lo:400:kworker.0 00000100:00100000:0.0:1767320561.885810:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457024 00000100:00100000:0.0:1767320561.885815:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457024 00000100:00100000:0.0:1767320561.885861:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457280 00000100:00100000:0.0:1767320561.885863:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457280 00000100:00100000:0.0:1767320561.885883:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5295 00000100:00100000:0.0:1767320561.885889:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985457024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320561.885928:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985457024:12345-0@lo:400:kworker.0 Request processed in 39us (993us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320561.885938:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5295 00000100:00100000:0.0:1767320561.885951:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5296 00000100:00100000:0.0:1767320561.885957:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985457280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320561.885996:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985457280:12345-0@lo:400:kworker.0 Request processed in 39us (998us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320561.886002:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5296 00000100:00100000:0.0:1767320561.886032:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985457024:0@lo:400:kworker.0 00000100:00080000:1.0:1767320561.886696:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320561.886744:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985457280:0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.887048:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985456768:0@lo:38:kworker.0 00000100:00100000:3.0:1767320561.887117:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320561.887202:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985456896:0@lo:38:kworker.0 00000100:00100000:3.0:1767320561.887231:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320561.887260:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985457152:0@lo:38:kworker.0 00000100:00100000:3.0:1767320561.887281:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320561.887301:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985457408:0@lo:38:kworker.0 00000100:00100000:3.0:1767320561.887326:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320561.887372:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456768 00000100:00100000:3.0:1767320561.887379:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456768 00000100:00100000:3.0:1767320561.887412:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457152 00000100:00100000:3.0:1767320561.887415:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457152 00000100:00100000:3.0:1767320561.887422:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457408 00000100:00100000:3.0:1767320561.887424:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457408 00000100:00100000:3.0:1767320561.887436:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5998 00000100:00100000:3.0:1767320561.887443:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985456768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320561.887468:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320561.887513:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985456768:12345-0@lo:38:kworker.0 Request processed in 69us (396us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320561.887520:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5998 00000100:00100000:1.0:1767320561.887521:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985456896 00000100:00100000:1.0:1767320561.887525:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985456896 00000100:00100000:3.0:1767320561.887536:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5999 00000100:00100000:3.0:1767320561.887540:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985457152:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320561.887546:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1990 00010000:00020000:3.0:1767320561.887549:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320561.887551:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985456896:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320561.887571:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985457152:12345-0@lo:38:kworker.0 Request processed in 29us (288us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320561.887576:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5999 00000100:00100000:3.0:1767320561.887588:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6000 00000100:00100000:3.0:1767320561.887591:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985457408:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320561.887599:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320561.887620:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985457408:12345-0@lo:38:kworker.0 Request processed in 27us (294us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320561.887626:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6000 00010000:00020000:1.0:1767320561.887637:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320561.887671:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985456896:12345-0@lo:38:kworker.0 Request processed in 119us (439us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767320561.887673:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320561.887678:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1990 00000100:00080000:3.0:1767320561.887679:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320561.887682:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985456768:0@lo:38:kworker.0 00000100:00080000:3.0:1767320561.887690:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320561.887693:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320561.887695:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985456896:0@lo:38:kworker.0 00000100:00080000:3.0:1767320561.887702:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320561.887705:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320561.887708:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985457152:0@lo:38:kworker.0 00000100:00080000:3.0:1767320561.887716:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320561.887720:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320561.887723:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985457408:0@lo:38:kworker.0 00010000:00010000:1.0:1767320562.523494:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320563.547998:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320564.443583:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985457792:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320564.443680:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320564.443880:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457792 00000100:00100000:3.0:1767320564.443891:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457792 00000100:00080000:1.0:1767320564.443948:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320564.443978:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2644 00000100:00100000:3.0:1767320564.443990:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985457792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320564.444029:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320564.444124:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320564.444137:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985457920:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320564.444189:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985457792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 201us (507us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320564.444199:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2644 00000100:00100000:1.0:1767320564.444221:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320564.444242:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985457920 00000100:00100000:3.0:1767320564.444246:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985457920 00000100:00100000:3.0:1767320564.444278:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2645 00000100:00100000:3.0:1767320564.444285:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985457920:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320564.444288:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:3.0:1767320564.444304:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320564.444340:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985457920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (119us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320564.444345:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2645 00000100:00100000:1.0:1767320564.444419:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985457920:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320564.444495:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985457792:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320564.573031:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320565.595842:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320566.625143:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320566.957333:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767320566.957378:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920624384 offset 0 length 520: 4 00000100:00100000:2.0:1767320566.957408:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320566.957447:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920624128 00000100:00100000:0.0:1767320566.957461:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920624128 00000100:00100000:0.0:1767320566.957516:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920624256 00000100:00100000:0.0:1767320566.957520:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920624256 00000100:00100000:0.0:1767320566.957574:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5297 00000100:00100000:0.0:1767320566.957592:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920624256:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320566.957688:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920624256:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 102us (277us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320566.957697:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5297 00000100:00100000:0.0:1767320566.957719:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6001 00000100:00100000:0.0:1767320566.957723:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920624128:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320566.957749:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320566.957775:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920624128:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (457us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320566.957783:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6001 00000100:00080000:2.0:1767320566.958813:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320567.003388:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320567.003410:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320567.003414:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320567.003419:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320567.003426:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320567.003436:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5810 00000100:00080000:2.0:1767320567.003442:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320567.003451:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320567.003498:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8700 x1853163985458048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320567.003525:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320567.003529:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320567.003531:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320567.003535:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5810 00000100:00080000:2.0:1767320567.003539:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320567.003542:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320567.003557:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e1500 x1853163985458176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320567.003605:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320567.003608:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320567.003610:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320567.003614:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5810 00000100:00080000:2.0:1767320567.003616:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320567.003619:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320567.003629:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3480 x1853163985458432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320567.003657:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320567.003661:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320567.003663:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320567.003666:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5810 00000100:00080000:2.0:1767320567.003669:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320567.003672:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320567.003680:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e0380 x1853163985458688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320567.003729:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985458560:0@lo:400:kworker.0 00000100:00100000:2.0:1767320567.003812:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320567.003899:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320567.003906:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985458816:0@lo:400:kworker.0 00000100:00100000:2.0:1767320567.003938:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320567.003950:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985458304:0@lo:400:kworker.0 00000100:00100000:2.0:1767320567.003994:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458560 00000100:00100000:2.0:1767320567.004004:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458560 00000100:00100000:3.0:1767320567.004064:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320567.004118:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320567.004126:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985458944:0@lo:400:kworker.0 00000100:00100000:3.0:1767320567.004157:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320567.004357:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458816 00000100:00100000:0.0:1767320567.004364:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458816 00000100:00100000:0.0:1767320567.004414:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3445 00000100:00100000:0.0:1767320567.004424:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985458816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320567.004491:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985458816:12345-0@lo:400:kworker.0 Request processed in 70us (552us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320567.004498:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3445 00000100:00100000:0.0:1767320567.004523:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458304 00000100:00100000:0.0:1767320567.004526:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458304 00000100:00100000:0.0:1767320567.004548:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5298 00000100:00100000:0.0:1767320567.004553:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985458560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320567.004582:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985458560:12345-0@lo:400:kworker.0 Request processed in 28us (770us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320567.004587:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5298 00000100:00100000:0.0:1767320567.004599:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5299 00000100:00100000:0.0:1767320567.004604:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985458304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320567.004625:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985458304:12345-0@lo:400:kworker.0 Request processed in 22us (567us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320567.004632:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5299 00000100:00100000:0.0:1767320567.004677:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985458816:0@lo:400:kworker.0 00000100:00100000:0.0:1767320567.004695:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985458560:0@lo:400:kworker.0 00000100:00100000:3.0:1767320567.004731:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985458304:0@lo:400:kworker.0 00000100:00080000:0.0:1767320567.004741:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320567.004748:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320567.004769:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458944 00000100:00100000:0.0:1767320567.004774:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458944 00000100:00080000:3.0:1767320567.004800:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320567.004806:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3446 00000100:00100000:0.0:1767320567.004814:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985458944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320567.004856:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985458944:12345-0@lo:400:kworker.0 Request processed in 43us (698us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320567.004861:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3446 00000100:00100000:1.0:1767320567.005677:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320541/real 1767320541] req@ffff8ce1b5b89180 x1853163985451520/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320566 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320567.005704:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5b89180 x1853163985451520/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320566 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320567.005725:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985458048:0@lo:38:kworker.0 00000100:00100000:1.0:1767320567.005777:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320567.005828:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985458176:0@lo:38:kworker.0 00000100:00100000:1.0:1767320567.005856:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320567.005889:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985458432:0@lo:38:kworker.0 00000100:00100000:1.0:1767320567.005913:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320567.005936:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985458688:0@lo:38:kworker.0 00000100:00100000:1.0:1767320567.005957:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320567.006020:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458176 00000100:00100000:0.0:1767320567.006028:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458176 00000100:00100000:0.0:1767320567.006066:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1991 00000100:00100000:0.0:1767320567.006074:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985458176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320567.006099:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320567.006120:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985458944:0@lo:400:kworker.0 00000100:00100000:0.0:1767320567.006151:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985458176:12345-0@lo:38:kworker.0 Request processed in 75us (291us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320567.006160:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1991 00000100:00080000:1.0:1767320567.017454:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320567.017501:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320567.017509:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985451520:0@lo:250:kworker.0 00000100:00080000:1.0:1767320567.017588:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320567.017593:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320567.017597:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985458176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320567.018798:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458048 00000100:00100000:2.0:1767320567.018813:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458048 00000100:00100000:2.0:1767320567.018898:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458432 00000100:00100000:2.0:1767320567.018903:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458432 00000100:00100000:2.0:1767320567.018920:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985458688 00000100:00100000:2.0:1767320567.018923:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985458688 00000100:00100000:2.0:1767320567.018941:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6002 00000100:00100000:2.0:1767320567.018951:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985458048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320567.018980:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320567.019445:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985458048:12345-0@lo:38:kworker.0 Request processed in 305us (13477us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320567.019456:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6002 00000100:00100000:2.0:1767320567.019481:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6003 00000100:00100000:2.0:1767320567.019486:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985458432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320567.019498:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320567.019531:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985458432:12345-0@lo:38:kworker.0 Request processed in 44us (13618us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320567.019538:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6003 00000100:00100000:2.0:1767320567.019549:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6004 00000100:00100000:2.0:1767320567.019553:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985458688:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320567.019562:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320567.019587:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985458688:12345-0@lo:38:kworker.0 Request processed in 32us (13629us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320567.019592:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6004 00000100:00080000:2.0:1767320567.020383:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320567.020389:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320567.020393:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320567.020401:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5789 00000100:00080000:2.0:1767320567.020407:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320567.020413:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320567.020450:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163985459072/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320567.020497:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985459072:0@lo:250:kworker.0 00000400:00000100:2.0:1767320567.020545:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985459072 offset 0 length 520: 4 00000100:00080000:2.0:1767320567.020596:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320567.020600:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320567.020605:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985458048:0@lo:38:kworker.0 00000100:00080000:2.0:1767320567.020616:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320567.020619:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320567.020622:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985458432:0@lo:38:kworker.0 00000100:00080000:2.0:1767320567.020630:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320567.020633:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320567.020636:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985458688:0@lo:38:kworker.0 00010000:00010000:1.0:1767320567.643410:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320568.669148:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320569.564187:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985459200:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320569.564259:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320569.564688:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985459200 00000100:00100000:1.0:1767320569.564698:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985459200 00000100:00100000:1.0:1767320569.564770:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2646 00000100:00100000:1.0:1767320569.564781:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985459200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320569.564822:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320569.564914:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985459200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 135us (653us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320569.564921:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2646 00000100:00080000:3.0:1767320569.565017:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320569.565339:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985459328:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320569.565454:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320569.565520:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985459328 00000100:00100000:2.0:1767320569.565527:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985459328 00000100:00100000:2.0:1767320569.565575:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2647 00000100:00100000:2.0:1767320569.565584:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985459328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320569.565606:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320569.565625:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985459200:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320569.565668:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985459328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (200us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320569.565677:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2647 00000100:00100000:0.0:1767320569.565681:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985459328:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320569.565750:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320569.691487:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320570.715563:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320571.739518:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320572.073875:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320572.073988:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320572.077088:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920624640 00000100:00100000:3.0:1767320572.077103:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920624640 00000100:00100000:3.0:1767320572.077198:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920624512 00000100:00100000:3.0:1767320572.077202:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920624512 00000100:00100000:2.0:1767320572.077253:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5300 00000100:00100000:2.0:1767320572.077266:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920624640:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320572.077367:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920624640:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (3503us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320572.077375:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5300 00000100:00100000:3.0:1767320572.077905:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6005 00000100:00100000:3.0:1767320572.077912:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920624512:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320572.077932:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320572.077966:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920624512:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (3977us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320572.077972:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6005 00000100:00080000:1.0:1767320572.078356:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320572.123396:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320572.123408:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320572.123411:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320572.123415:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320572.123420:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320572.123428:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5815 00000100:00080000:0.0:1767320572.123434:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320572.123441:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320572.123480:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594e00 x1853163985459456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320572.123509:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320572.123513:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320572.123514:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320572.123518:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5815 00000100:00080000:0.0:1767320572.123521:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320572.123524:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320572.123537:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595c00 x1853163985459584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320572.123624:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985459712:0@lo:400:kworker.0 00000100:00100000:1.0:1767320572.123690:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320572.123864:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320572.123869:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320572.123871:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320572.123875:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5815 00000100:00080000:0.0:1767320572.123877:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320572.123880:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320572.123896:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596300 x1853163985459840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320572.123919:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320572.123922:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320572.123924:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320572.123927:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5815 00000100:00080000:0.0:1767320572.123930:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320572.123933:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320572.123942:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34e3480 x1853163985460096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320572.124006:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985459456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320572.124035:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320572.124059:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985459584:0@lo:38:kworker.0 00000100:00100000:2.0:1767320572.124084:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320572.124115:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985459840:0@lo:38:kworker.0 00000100:00100000:3.0:1767320572.124115:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985459712 00000100:00100000:3.0:1767320572.124127:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985459712 00000100:00100000:2.0:1767320572.124138:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320572.124159:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985460096:0@lo:38:kworker.0 00000100:00100000:2.0:1767320572.124204:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320572.124219:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985459456 00000100:00100000:3.0:1767320572.124224:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985459456 00000100:00100000:3.0:1767320572.124247:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985459840 00000100:00100000:3.0:1767320572.124251:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985459840 00000100:00100000:2.0:1767320572.124253:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460096 00000100:00100000:2.0:1767320572.124259:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460096 00000100:00100000:3.0:1767320572.124269:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5301 00000100:00100000:3.0:1767320572.124281:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985459712:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124286:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6006 00000100:00100000:2.0:1767320572.124292:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985459456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320572.124308:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320572.124350:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985459456:12345-0@lo:38:kworker.0 Request processed in 56us (314us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320572.124360:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6006 00000100:00100000:2.0:1767320572.124376:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6007 00000100:00100000:2.0:1767320572.124380:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985459840:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320572.124389:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320572.124411:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985459840:12345-0@lo:38:kworker.0 Request processed in 30us (274us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320572.124417:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6007 00000100:00100000:2.0:1767320572.124428:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6008 00000100:00100000:2.0:1767320572.124431:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985460096:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320572.124442:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320572.124464:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985460096:12345-0@lo:38:kworker.0 Request processed in 30us (261us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320572.124469:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6008 00000100:00080000:2.0:1767320572.124509:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320572.124514:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320572.124518:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985459456:0@lo:38:kworker.0 00000100:00080000:2.0:1767320572.124527:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320572.124529:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320572.124530:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985459840:0@lo:38:kworker.0 00000100:00080000:2.0:1767320572.124535:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320572.124536:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320572.124538:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985460096:0@lo:38:kworker.0 00000100:00080000:2.0:1767320572.124576:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320572.124588:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985460352:0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124606:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320572.124628:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985459968:0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124649:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320572.124666:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320572.124671:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985460224:0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124692:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320572.124712:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985459968 00000100:00100000:2.0:1767320572.124715:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985459968 00000100:00100000:2.0:1767320572.124746:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5302 00000100:00100000:2.0:1767320572.124750:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985459968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124772:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985459968:12345-0@lo:400:kworker.0 Request processed in 22us (123us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320572.124776:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5302 00000100:00080000:2.0:1767320572.124803:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320572.124823:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985459968:0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124847:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460352 00000100:00100000:2.0:1767320572.124850:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460352 00000100:00100000:2.0:1767320572.124877:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460224 00000100:00100000:2.0:1767320572.124880:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460224 00000100:00100000:2.0:1767320572.124900:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3447 00000100:00100000:2.0:1767320572.124906:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985460352:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124937:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985460352:12345-0@lo:400:kworker.0 Request processed in 31us (329us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320572.124942:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3447 00000100:00100000:2.0:1767320572.124953:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3448 00000100:00100000:2.0:1767320572.124960:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985460224:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.124993:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985460224:12345-0@lo:400:kworker.0 Request processed in 34us (300us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320572.124998:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3448 00000100:00100000:2.0:1767320572.125021:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985460352:0@lo:400:kworker.0 00000100:00080000:2.0:1767320572.125034:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320572.125037:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320572.125049:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985460224:0@lo:400:kworker.0 00000100:00100000:2.0:1767320572.125094:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985459584 00000100:00100000:2.0:1767320572.125098:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985459584 00000100:00100000:2.0:1767320572.125118:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1992 00000100:00100000:2.0:1767320572.125122:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985459584:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320572.125133:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320572.125177:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985459584:12345-0@lo:38:kworker.0 Request processed in 38us (1078us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320572.125188:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1992 00000100:00080000:2.0:1767320572.125215:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320572.125219:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320572.125222:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985459584:0@lo:38:kworker.0 00000100:00100000:3.0:1767320572.128121:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985459712:12345-0@lo:400:kworker.0 Request processed in 3831us (4419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320572.128139:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5301 00000100:00100000:1.0:1767320572.135880:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985459712:0@lo:400:kworker.0 00010000:00010000:1.0:1767320572.763522:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320573.788157:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320574.684878:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320574.684903:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985460480:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320574.684969:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320574.685150:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460480 00000100:00100000:0.0:1767320574.685158:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460480 00000100:00100000:0.0:1767320574.685235:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2648 00000100:00100000:0.0:1767320574.685244:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985460480:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320574.685274:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320574.685325:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985460480:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (355us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320574.685333:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2648 00000100:00080000:3.0:1767320574.685619:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320574.685657:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985460480:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320574.685875:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985460608:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320574.685914:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320574.685997:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460608 00000100:00100000:3.0:1767320574.686004:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460608 00000100:00100000:3.0:1767320574.686049:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2649 00000100:00100000:3.0:1767320574.686055:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985460608:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320574.686073:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320574.686103:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985460608:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (189us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320574.686109:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2649 00000100:00080000:2.0:1767320574.686266:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320574.686290:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985460608:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320574.813525:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320575.835660:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320576.859367:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320577.195157:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320577.195362:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320577.203460:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920624896 00000100:00100000:2.0:1767320577.203478:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920624896 00000100:00100000:2.0:1767320577.203707:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920624768 00000100:00100000:2.0:1767320577.203712:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920624768 00000100:00100000:2.0:1767320577.203742:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5303 00000100:00100000:2.0:1767320577.203756:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920624896:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320577.203936:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6009 00000100:00100000:1.0:1767320577.203944:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920624768:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320577.203964:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320577.204033:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920624768:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 88us (8671us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320577.204041:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6009 00000100:00100000:2.0:1767320577.206717:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920624896:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 2949us (11554us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320577.206736:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5303 00000100:00080000:0.0:1767320577.215796:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320577.244477:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320577.244487:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320577.244490:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320577.244494:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320577.244498:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320577.244505:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320577.244511:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320577.244517:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320577.244553:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595c00 x1853163985460736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320577.244586:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320577.244589:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320577.244591:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320577.244595:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320577.244597:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320577.244600:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320577.244617:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc000 x1853163985460864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320577.244654:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320577.244658:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320577.244659:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320577.244663:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320577.244666:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320577.244670:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320577.244683:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffe300 x1853163985461120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320577.244726:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320577.244730:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320577.244732:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320577.244737:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320577.244740:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320577.244744:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320577.244755:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985461376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320577.244802:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985461632:0@lo:400:kworker.0 00000100:00100000:2.0:1767320577.244865:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320577.244921:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320577.244927:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985460992:0@lo:400:kworker.0 00000100:00100000:2.0:1767320577.244950:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320577.245066:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985461504:0@lo:400:kworker.0 00000100:00100000:2.0:1767320577.245091:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320577.245110:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320577.245114:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985461248:0@lo:400:kworker.0 00000100:00100000:2.0:1767320577.245131:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320577.245157:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985461632 00000100:00100000:2.0:1767320577.245188:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985461632 00000100:00100000:2.0:1767320577.245237:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985461504 00000100:00100000:2.0:1767320577.245241:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985461504 00000100:00100000:2.0:1767320577.245265:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3449 00000100:00100000:2.0:1767320577.245273:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985461632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320577.245329:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985461632:12345-0@lo:400:kworker.0 Request processed in 56us (462us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320577.245475:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3449 00000100:00100000:3.0:1767320577.247152:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460992 00000100:00100000:3.0:1767320577.247211:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460992 00000100:00100000:3.0:1767320577.247285:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985461248 00000100:00100000:3.0:1767320577.247289:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985461248 00000100:00100000:3.0:1767320577.247315:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5304 00000100:00100000:3.0:1767320577.247325:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985460992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320577.247441:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985460992:12345-0@lo:400:kworker.0 Request processed in 118us (2489us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320577.247456:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5304 00000100:00100000:3.0:1767320577.247474:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5305 00000100:00100000:3.0:1767320577.247487:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985461248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320577.247517:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985461248:12345-0@lo:400:kworker.0 Request processed in 31us (2385us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320577.247532:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5305 00000100:00100000:3.0:1767320577.248744:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985460992:0@lo:400:kworker.0 00000100:00100000:3.0:1767320577.248765:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985461632:0@lo:400:kworker.0 00000100:00100000:3.0:1767320577.249250:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985461248:0@lo:400:kworker.0 00000100:00100000:1.0:1767320577.249290:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3450 00000100:00100000:1.0:1767320577.249321:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985461504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320577.249335:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985460736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320577.249439:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985461504:12345-0@lo:400:kworker.0 Request processed in 132us (4343us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320577.249448:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3450 00000100:00100000:1.0:1767320577.249515:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985461504:0@lo:400:kworker.0 00000100:00080000:1.0:1767320577.249559:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320577.249565:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320577.249567:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320577.253946:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320577.254891:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985460864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320577.254944:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320577.255072:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460736 00000100:00100000:3.0:1767320577.255090:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460736 00000100:00100000:3.0:1767320577.255150:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6010 00000100:00100000:3.0:1767320577.255161:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985460736:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320577.255190:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985460864 00000100:00100000:1.0:1767320577.255199:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985460864 00010000:00020000:3.0:1767320577.255215:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320577.255242:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1993 00000100:00100000:1.0:1767320577.255252:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985460864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320577.255275:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320577.255294:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985460736:12345-0@lo:38:kworker.0 Request processed in 133us (1354us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320577.255305:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6010 00000100:00100000:1.0:1767320577.255329:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985460864:12345-0@lo:38:kworker.0 Request processed in 75us (381us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320577.255338:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1993 00000100:00080000:1.0:1767320577.255389:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320577.255394:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320577.255397:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985460864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320577.255406:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985461120:0@lo:38:kworker.0 00000100:00100000:1.0:1767320577.255430:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320577.255452:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985461376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320577.255474:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320577.255497:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985461120 00000100:00100000:1.0:1767320577.255501:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985461120 00000100:00100000:1.0:1767320577.255521:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985461376 00000100:00100000:1.0:1767320577.255524:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985461376 00000100:00100000:1.0:1767320577.255540:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6011 00000100:00100000:1.0:1767320577.255544:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985461120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320577.255555:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320577.255579:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985461120:12345-0@lo:38:kworker.0 Request processed in 33us (148us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320577.255585:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6011 00000100:00100000:1.0:1767320577.255596:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6012 00000100:00100000:1.0:1767320577.255600:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985461376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320577.255609:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320577.255629:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985461376:12345-0@lo:38:kworker.0 Request processed in 27us (154us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320577.255638:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6012 00000100:00080000:1.0:1767320577.255667:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320577.255671:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320577.255675:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985461376:0@lo:38:kworker.0 00000100:00080000:1.0:1767320577.255718:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320577.255722:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320577.255725:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985460736:0@lo:38:kworker.0 00000100:00080000:1.0:1767320577.255733:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320577.255735:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320577.255738:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985461120:0@lo:38:kworker.0 00010000:00010000:2.0:1767320577.884045:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320578.907378:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320579.803785:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320579.803808:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985461760:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320579.803871:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320579.804031:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985461760 00000100:00100000:2.0:1767320579.804040:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985461760 00000100:00100000:2.0:1767320579.804100:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2650 00000100:00100000:2.0:1767320579.804108:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985461760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320579.804136:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320579.804195:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985461760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (323us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320579.804202:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2650 00000100:00080000:2.0:1767320579.804224:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320579.804324:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985461760:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320579.804782:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320579.804793:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985461888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320579.804822:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320579.804892:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985461888 00000100:00100000:1.0:1767320579.804898:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985461888 00000100:00080000:2.0:1767320579.805021:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320579.805059:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2651 00000100:00100000:1.0:1767320579.805068:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985461888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320579.805086:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320579.805128:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985461888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320579.805135:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2651 00000100:00100000:2.0:1767320579.805212:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985461888:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320579.931675:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320580.960833:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320581.979409:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320582.317199:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320582.317249:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320582.317330:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920625152 00000100:00100000:3.0:1767320582.317339:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920625152 00000100:00100000:3.0:1767320582.317412:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920625024 00000100:00100000:3.0:1767320582.317415:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920625024 00000100:00100000:3.0:1767320582.317435:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5306 00000100:00100000:3.0:1767320582.317445:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920625152:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320582.317518:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920625152:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320582.317530:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5306 00000100:00100000:3.0:1767320582.317549:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6013 00000100:00100000:3.0:1767320582.317554:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920625024:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320582.317577:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320582.317618:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920625024:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 62us (369us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320582.317624:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6013 00000100:00080000:2.0:1767320582.318114:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320582.363337:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.363347:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320582.363360:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.363364:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.363368:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.363376:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.363380:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.363386:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320582.363426:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe23800 x1853163985462016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320582.363457:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.363460:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.363461:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.363465:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.363467:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.363470:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320582.363479:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21880 x1853163985462144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320582.363516:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.363519:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.363521:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.363524:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.363527:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.363529:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320582.363539:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22680 x1853163985462400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320582.363573:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.363576:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.363578:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.363581:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.363583:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.363586:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320582.363603:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe23480 x1853163985462656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320582.363643:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985462528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.363705:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320582.363762:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320582.363777:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985462784:0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.363799:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320582.363825:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462528 00000100:00100000:2.0:1767320582.363832:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462528 00000100:00100000:2.0:1767320582.363880:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5307 00000100:00100000:2.0:1767320582.363888:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985462528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.363937:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985462528:12345-0@lo:400:kworker.0 Request processed in 49us (231us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320582.363944:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5307 00000100:00080000:2.0:1767320582.363963:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320582.363987:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985462528:0@lo:400:kworker.0 00000100:00100000:3.0:1767320582.364104:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985462272:0@lo:400:kworker.0 00000100:00100000:3.0:1767320582.364142:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320582.364175:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320582.364180:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985462912:0@lo:400:kworker.0 00000100:00100000:3.0:1767320582.364198:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320582.364405:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462784 00000100:00100000:2.0:1767320582.364410:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462784 00000100:00100000:2.0:1767320582.364445:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462912 00000100:00100000:2.0:1767320582.364447:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462912 00000100:00100000:2.0:1767320582.364462:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3451 00000100:00100000:2.0:1767320582.364468:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985462784:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.364500:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985462784:12345-0@lo:400:kworker.0 Request processed in 32us (699us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320582.364505:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3451 00000100:00100000:2.0:1767320582.364515:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3452 00000100:00100000:2.0:1767320582.364520:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985462912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.364541:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985462912:12345-0@lo:400:kworker.0 Request processed in 22us (342us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320582.364547:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3452 00000100:00100000:2.0:1767320582.364578:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985462912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.364602:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462272 00000100:00100000:2.0:1767320582.364604:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462272 00000100:00100000:2.0:1767320582.364625:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5308 00000100:00100000:2.0:1767320582.364630:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985462272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.364657:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985462272:12345-0@lo:400:kworker.0 Request processed in 26us (514us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320582.364662:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5308 00000100:00100000:2.0:1767320582.364686:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985462272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.364702:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985462784:0@lo:400:kworker.0 00000100:00080000:3.0:1767320582.365020:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320582.365024:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320582.365026:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320582.366711:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985462016:0@lo:38:kworker.0 00000100:00100000:1.0:1767320582.366781:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320582.366824:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985462144:0@lo:38:kworker.0 00000100:00100000:1.0:1767320582.366849:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320582.366884:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985462400:0@lo:38:kworker.0 00000100:00100000:1.0:1767320582.366904:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320582.366934:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985462656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320582.366953:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320582.366993:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462016 00000100:00100000:1.0:1767320582.367011:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462016 00000100:00100000:1.0:1767320582.367040:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462400 00000100:00100000:1.0:1767320582.367043:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462400 00000100:00100000:1.0:1767320582.367054:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462656 00000100:00100000:1.0:1767320582.367056:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462656 00000100:00100000:1.0:1767320582.367068:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6014 00000100:00100000:1.0:1767320582.367074:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985462016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320582.367095:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320582.367132:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985462016:12345-0@lo:38:kworker.0 Request processed in 56us (349us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320582.367138:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6014 00000100:00100000:1.0:1767320582.367153:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6015 00000100:00100000:1.0:1767320582.367157:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985462400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320582.367185:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320582.367206:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985462400:12345-0@lo:38:kworker.0 Request processed in 48us (301us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320582.367212:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6015 00000100:00100000:1.0:1767320582.367223:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6016 00000100:00100000:1.0:1767320582.367227:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985462656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320582.367236:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320582.367256:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985462656:12345-0@lo:38:kworker.0 Request processed in 28us (303us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320582.367261:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6016 00000100:00080000:1.0:1767320582.367294:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320582.367298:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320582.367302:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985462016:0@lo:38:kworker.0 00000100:00080000:1.0:1767320582.367311:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320582.367314:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320582.367317:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985462400:0@lo:38:kworker.0 00000100:00080000:1.0:1767320582.367324:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320582.367327:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320582.367330:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985462656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320582.367750:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985462144 00000100:00100000:2.0:1767320582.367756:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985462144 00000100:00100000:2.0:1767320582.367786:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1994 00000100:00100000:2.0:1767320582.367793:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985462144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320582.367805:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320582.367848:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985462144:12345-0@lo:38:kworker.0 Request processed in 53us (997us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320582.367854:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1994 00000100:00080000:3.0:1767320582.367932:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320582.367937:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320582.367941:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985462144:0@lo:38:kworker.0 00010000:00010000:2.0:1767320583.003604:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320584.035581:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320584.923399:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320584.923424:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985463168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320584.923447:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320584.923460:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985463040:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320584.923811:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320584.924043:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463168 00000100:00100000:2.0:1767320584.924052:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463168 00000100:00100000:2.0:1767320584.924180:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2652 00000100:00100000:2.0:1767320584.924190:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985463168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320584.924219:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320584.924268:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985463168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (457us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320584.924275:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2652 00000100:00100000:2.0:1767320584.924323:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985463168:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320584.924674:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320584.928431:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320584.928737:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463040 00000100:00100000:1.0:1767320584.928750:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463040 00000100:00100000:1.0:1767320584.928840:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2653 00000100:00100000:1.0:1767320584.928851:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985463040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320584.928886:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320584.928956:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985463040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 106us (531us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320584.928964:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2653 00000100:00080000:3.0:1767320584.929014:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320584.930902:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985463040:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320585.051426:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320586.075516:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320587.099405:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320587.433784:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320587.433943:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920625408 00000100:00100000:2.0:1767320587.433953:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920625408 00000100:00100000:2.0:1767320587.434033:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5309 00000100:00100000:2.0:1767320587.434046:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920625408:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320587.434128:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920625408:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (358us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320587.434136:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5309 00000100:00080000:3.0:1767320587.434406:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320587.435048:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320587.435119:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920625280 00000100:00100000:2.0:1767320587.435125:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920625280 00000100:00100000:2.0:1767320587.435156:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6017 00000100:00100000:2.0:1767320587.435192:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920625280:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320587.435213:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320587.435260:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920625280:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 96us (214us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320587.435266:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6017 00000100:00080000:3.0:1767320587.483439:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320587.483447:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320587.483449:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320587.483452:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320587.483456:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320587.483462:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5830 00000100:00080000:3.0:1767320587.483466:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320587.483471:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320587.483518:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06d80 x1853163985463296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320587.483542:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320587.483545:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320587.483548:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320587.483551:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5830 00000100:00080000:3.0:1767320587.483553:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320587.483557:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320587.483568:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985463424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320587.483620:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985463552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320587.483683:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.483804:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463552 00000100:00100000:0.0:1767320587.483819:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463552 00000100:00100000:1.0:1767320587.483819:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985463296:0@lo:38:kworker.0 00000100:00100000:1.0:1767320587.483855:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320587.483891:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985463424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320587.483914:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320587.483950:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463424 00000100:00100000:1.0:1767320587.483954:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463424 00000100:00100000:1.0:1767320587.483980:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1995 00000100:00100000:1.0:1767320587.483986:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985463424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320587.484006:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320587.484042:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985463424:12345-0@lo:38:kworker.0 Request processed in 55us (127us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320587.484049:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1995 00000100:00080000:1.0:1767320587.484084:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320587.484089:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320587.484092:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985463424:0@lo:38:kworker.0 00000100:00080000:1.0:1767320587.484122:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320587.485705:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320587.485711:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320587.485714:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320587.485718:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5830 00000100:00080000:3.0:1767320587.485722:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320587.485726:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320587.485758:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05180 x1853163985463680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320587.485824:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320587.485827:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320587.485828:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320587.485831:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5830 00000100:00080000:3.0:1767320587.485833:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320587.485835:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320587.485844:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04a80 x1853163985463936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320587.486162:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320587.486184:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985464192:0@lo:400:kworker.0 00000100:00100000:2.0:1767320587.486232:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320587.486271:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464192 00000100:00100000:2.0:1767320587.486277:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464192 00000100:00100000:2.0:1767320587.486321:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3453 00000100:00100000:2.0:1767320587.486329:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985464192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.486587:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463296 00000100:00100000:0.0:1767320587.486592:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463296 00000100:00100000:0.0:1767320587.486612:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5310 00000100:00100000:0.0:1767320587.486620:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985463552:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.486870:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985463552:12345-0@lo:400:kworker.0 Request processed in 250us (3183us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320587.486879:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5310 00000100:00100000:0.0:1767320587.486898:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6018 00000100:00100000:0.0:1767320587.486902:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985463296:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320587.486921:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320587.486952:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985463296:12345-0@lo:38:kworker.0 Request processed in 48us (3097us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320587.486958:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6018 00000100:00100000:0.0:1767320587.486981:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985463808:0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.487009:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.487028:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320587.487035:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985464064:0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.487057:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.487099:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463808 00000100:00100000:0.0:1767320587.487103:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463808 00000100:00100000:0.0:1767320587.487142:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5311 00000100:00100000:0.0:1767320587.487149:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985463808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.487196:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985463808:12345-0@lo:400:kworker.0 Request processed in 48us (186us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320587.487202:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5311 00000100:00100000:0.0:1767320587.487238:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985463808:0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.487282:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985463680:0@lo:38:kworker.0 00000100:00100000:0.0:1767320587.487320:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.487336:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985463936:0@lo:38:kworker.0 00000100:00100000:0.0:1767320587.487355:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320587.487378:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320587.487382:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320587.487386:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985463296:0@lo:38:kworker.0 00000100:00100000:0.0:1767320587.487407:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463680 00000100:00100000:0.0:1767320587.487411:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463680 00000100:00100000:0.0:1767320587.487428:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985463936 00000100:00100000:0.0:1767320587.487430:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985463936 00000100:00100000:0.0:1767320587.487443:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6019 00000100:00100000:0.0:1767320587.487448:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985463680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320587.487460:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320587.487487:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985463680:12345-0@lo:38:kworker.0 Request processed in 38us (167us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320587.487494:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6019 00000100:00100000:0.0:1767320587.487518:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6020 00000100:00100000:0.0:1767320587.487523:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985463936:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320587.487532:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320587.487554:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985463936:12345-0@lo:38:kworker.0 Request processed in 30us (198us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320587.487560:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6020 00000100:00080000:0.0:1767320587.487583:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320587.487587:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320587.487590:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985463680:0@lo:38:kworker.0 00000100:00080000:0.0:1767320587.487597:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320587.487600:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320587.487603:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985463936:0@lo:38:kworker.0 00000100:00100000:0.0:1767320587.487635:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464064 00000100:00100000:0.0:1767320587.487638:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464064 00000100:00100000:0.0:1767320587.487662:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3454 00000100:00100000:0.0:1767320587.487669:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985464064:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.487702:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985464064:12345-0@lo:400:kworker.0 Request processed in 33us (643us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320587.487708:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3454 00000100:00100000:0.0:1767320587.487730:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985464064:0@lo:400:kworker.0 00000100:00080000:0.0:1767320587.487745:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320587.487749:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320587.487751:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320587.488073:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985464192:12345-0@lo:400:kworker.0 Request processed in 1744us (1838us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320587.488081:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3453 00000100:00100000:2.0:1767320587.488114:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985464192:0@lo:400:kworker.0 00000100:00100000:2.0:1767320587.488124:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985463552:0@lo:400:kworker.0 00010000:00010000:1.0:1767320588.123385:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320589.148748:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320590.045443:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985464320:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320590.045546:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320590.045701:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464320 00000100:00100000:1.0:1767320590.045711:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464320 00000100:00100000:1.0:1767320590.045795:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2654 00000100:00100000:1.0:1767320590.045806:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985464320:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320590.045847:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320590.045898:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985464320:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (352us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320590.045905:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2654 00000100:00080000:2.0:1767320590.046545:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320590.046590:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985464320:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320590.047259:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320590.047272:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985464448:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320590.047318:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320590.047424:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464448 00000100:00100000:3.0:1767320590.047431:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464448 00000100:00100000:3.0:1767320590.047492:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2655 00000100:00100000:3.0:1767320590.047501:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985464448:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320590.047527:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320590.047574:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985464448:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (255us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320590.047581:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2655 00000100:00100000:3.0:1767320590.047630:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985464448:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320590.048388:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320590.185419:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320591.195571:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320591.531763:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767320591.531809:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920625536 offset 0 length 520: 4 00000100:00100000:1.0:1767320591.531833:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320591.532130:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920625792 00000100:00100000:3.0:1767320591.532141:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920625792 00000100:00100000:3.0:1767320591.532247:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920625664 00000100:00100000:3.0:1767320591.532251:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920625664 00000100:00100000:3.0:1767320591.532274:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5312 00000100:00100000:3.0:1767320591.532284:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920625792:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320591.532361:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920625792:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (609us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320591.532373:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5312 00000100:00100000:3.0:1767320591.532392:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6021 00000100:00100000:3.0:1767320591.532397:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920625664:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320591.532416:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320591.532449:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920625664:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (615us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320591.532456:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6021 00000100:00080000:2.0:1767320591.533024:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320592.223900:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320592.603330:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320592.603352:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320592.603355:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320592.603359:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320592.603364:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320592.603374:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5835 00000100:00080000:2.0:1767320592.603380:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320592.603386:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320592.603438:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351aa00 x1853163985464576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320592.603463:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320592.603466:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320592.603468:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320592.603471:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5835 00000100:00080000:2.0:1767320592.603473:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320592.603476:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320592.603498:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519c00 x1853163985464704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320592.603546:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320592.603550:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320592.603552:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320592.603555:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5835 00000100:00080000:2.0:1767320592.603558:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320592.603561:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320592.603572:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b800 x1853163985464960/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320592.603595:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320592.603599:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320592.603600:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320592.603604:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5835 00000100:00080000:2.0:1767320592.603606:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320592.603609:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320592.603623:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518000 x1853163985465216/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320592.603660:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985465344:0@lo:400:kworker.0 00000100:00100000:0.0:1767320592.603760:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320567/real 1767320567] req@ffff8ce1a6a61f80 x1853163985459072/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320592 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320592.603826:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985465472:0@lo:400:kworker.0 00000100:00100000:3.0:1767320592.603909:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985464832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320592.607050:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a6a61f80 x1853163985459072/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320592 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320592.607182:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985464576:0@lo:38:kworker.0 00000100:00100000:0.0:1767320592.607338:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320592.607562:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985464704:0@lo:38:kworker.0 00000100:00100000:0.0:1767320592.607591:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320592.607620:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985464960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320592.607641:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320592.607657:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985465216:0@lo:38:kworker.0 00000100:00100000:0.0:1767320592.607677:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320592.607703:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320592.607721:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320592.607725:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985459072:0@lo:250:kworker.0 00000100:00100000:0.0:1767320592.607793:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464576 00000100:00100000:0.0:1767320592.607799:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464576 00000100:00100000:0.0:1767320592.607825:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464960 00000100:00100000:0.0:1767320592.607827:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464960 00000100:00100000:0.0:1767320592.607837:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985465216 00000100:00100000:0.0:1767320592.607839:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985465216 00000100:00100000:0.0:1767320592.607859:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6022 00000100:00100000:0.0:1767320592.607865:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985464576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320592.607884:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320592.607922:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985464576:12345-0@lo:38:kworker.0 Request processed in 56us (585us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320592.607929:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6022 00000100:00100000:0.0:1767320592.607942:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6023 00000100:00100000:0.0:1767320592.607946:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985464960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320592.607954:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320592.607972:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985464960:12345-0@lo:38:kworker.0 Request processed in 25us (330us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320592.607978:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6023 00000100:00100000:0.0:1767320592.607986:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6024 00000100:00100000:0.0:1767320592.607989:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985465216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320592.607997:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320592.608012:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985465216:12345-0@lo:38:kworker.0 Request processed in 22us (335us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320592.608017:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6024 00000100:00100000:0.0:1767320592.608032:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464704 00000100:00100000:0.0:1767320592.608035:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464704 00000100:00100000:0.0:1767320592.608051:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1996 00000100:00100000:0.0:1767320592.608055:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985464704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320592.608063:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320592.608356:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985464704:12345-0@lo:38:kworker.0 Request processed in 298us (762us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320592.608362:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1996 00000100:00080000:0.0:1767320592.609002:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320592.609009:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320592.609013:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985464576:0@lo:38:kworker.0 00000100:00080000:0.0:1767320592.609158:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320592.609162:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320592.609185:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985464704:0@lo:38:kworker.0 00000100:00080000:0.0:1767320592.609195:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320592.609198:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320592.609201:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985464960:0@lo:38:kworker.0 00000100:00080000:0.0:1767320592.609209:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320592.609212:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320592.609214:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985465216:0@lo:38:kworker.0 00000100:00100000:0.0:1767320592.609813:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985465088:0@lo:400:kworker.0 00000100:00100000:0.0:1767320592.609855:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320592.609896:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985465088 00000100:00100000:0.0:1767320592.609902:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985465088 00000100:00100000:0.0:1767320592.609952:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5313 00000100:00100000:0.0:1767320592.609960:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985465088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320592.609999:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985465088:12345-0@lo:400:kworker.0 Request processed in 40us (143us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320592.610006:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5313 00000100:00100000:0.0:1767320592.610047:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985465088:0@lo:400:kworker.0 00000100:00080000:0.0:1767320592.610091:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320592.610102:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320592.610107:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320592.610110:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320592.610114:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5815 00000100:00080000:0.0:1767320592.610117:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320592.610122:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320592.610140:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfa680 x1853163985465600/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320592.610192:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985465600:0@lo:250:kworker.0 00000400:00000100:0.0:1767320592.610212:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985465600 offset 0 length 520: 4 00000100:00100000:1.0:1767320592.611793:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320592.611978:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320592.612071:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985465472 00000100:00100000:1.0:1767320592.612078:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985465472 00000100:00100000:1.0:1767320592.612132:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985465344 00000100:00100000:1.0:1767320592.612135:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985465344 00000100:00100000:1.0:1767320592.612184:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3455 00000100:00100000:1.0:1767320592.612193:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985465472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320592.612251:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985465472:12345-0@lo:400:kworker.0 Request processed in 59us (462us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320592.612258:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3455 00000100:00100000:1.0:1767320592.612274:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3456 00000100:00100000:1.0:1767320592.612281:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985465344:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320592.612310:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985465344:12345-0@lo:400:kworker.0 Request processed in 30us (332us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320592.612316:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3456 00000100:00080000:2.0:1767320592.612587:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320592.612591:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320592.612621:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985465344:0@lo:400:kworker.0 00000100:00100000:1.0:1767320592.612892:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985465472:0@lo:400:kworker.0 00000100:00100000:3.0:1767320592.616294:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320592.616652:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985464832 00000100:00100000:2.0:1767320592.616663:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985464832 00000100:00100000:2.0:1767320592.616752:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5314 00000100:00100000:2.0:1767320592.616763:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985464832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320592.616839:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985464832:12345-0@lo:400:kworker.0 Request processed in 78us (553us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320592.616849:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5314 00000100:00080000:3.0:1767320592.619815:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320592.619881:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985464832:0@lo:400:kworker.0 00010000:00010000:1.0:1767320593.243752:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320594.267479:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320595.163523:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320595.163553:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985465728:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320595.163634:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320595.164457:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985465728 00000100:00100000:3.0:1767320595.164468:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985465728 00000100:00100000:3.0:1767320595.164546:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2656 00000100:00100000:3.0:1767320595.164556:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985465728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320595.164591:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320595.164656:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985465728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (1021us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320595.164664:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2656 00000100:00100000:3.0:1767320595.164715:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985465728:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320595.164789:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320595.173368:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985465856:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320595.173444:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320595.174861:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985465856 00000100:00100000:3.0:1767320595.174874:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985465856 00000100:00100000:3.0:1767320595.174967:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2657 00000100:00100000:3.0:1767320595.174978:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985465856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320595.175011:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767320595.175116:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320595.184602:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985465856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 9613us (11144us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320595.184622:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2657 00000100:00100000:0.0:1767320595.186113:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985465856:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320595.295781:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320596.315398:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320596.652825:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320596.652882:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320596.653064:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920626048 00000100:00100000:3.0:1767320596.653073:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920626048 00000100:00100000:3.0:1767320596.653141:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920625920 00000100:00100000:3.0:1767320596.653144:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920625920 00000100:00100000:3.0:1767320596.653186:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5315 00000100:00100000:3.0:1767320596.653195:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920626048:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320596.653254:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920626048:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 61us (440us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320596.653259:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5315 00000100:00100000:3.0:1767320596.653273:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6025 00000100:00100000:3.0:1767320596.653276:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920625920:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320596.653291:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320596.653306:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920625920:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 29us (424us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320596.653310:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6025 00000100:00080000:1.0:1767320596.653365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320597.344959:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320597.723339:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320597.723350:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320597.723354:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320597.723358:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320597.723364:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320597.723373:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5840 00000100:00080000:2.0:1767320597.723377:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320597.723383:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320597.723486:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfbb80 x1853163985465984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320597.723514:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320597.723518:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320597.723520:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320597.723523:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5840 00000100:00080000:0.0:1767320597.723525:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320597.723529:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320597.723542:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfad80 x1853163985466112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320597.723591:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320597.723594:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320597.723596:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320597.723599:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5840 00000100:00080000:0.0:1767320597.723601:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320597.723605:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320597.723616:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163985466368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320597.723642:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320597.723646:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320597.723647:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320597.723650:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5840 00000100:00080000:0.0:1767320597.723652:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320597.723656:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320597.723665:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c000 x1853163985466624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320597.723697:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985466752:0@lo:400:kworker.0 00000100:00100000:1.0:1767320597.723726:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985466240:0@lo:400:kworker.0 00000100:00100000:0.0:1767320597.723757:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320597.723772:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320597.723797:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985466752 00000100:00100000:0.0:1767320597.723805:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985466752 00000100:00100000:1.0:1767320597.723831:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320597.723840:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985466880:0@lo:400:kworker.0 00000100:00100000:0.0:1767320597.723863:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3457 00000100:00100000:1.0:1767320597.723868:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320597.723875:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985466752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320597.723901:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985466240 00000100:00100000:1.0:1767320597.723907:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985466240 00000100:00100000:0.0:1767320597.723924:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985466752:12345-0@lo:400:kworker.0 Request processed in 53us (167us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320597.723932:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3457 00000100:00100000:1.0:1767320597.723946:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5316 00000100:00100000:0.0:1767320597.723951:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985466880 00000100:00100000:0.0:1767320597.723957:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985466880 00000100:00100000:1.0:1767320597.723957:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985466240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320597.723988:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3458 00000100:00100000:0.0:1767320597.723997:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985466880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320597.724013:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985466240:12345-0@lo:400:kworker.0 Request processed in 49us (232us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320597.724021:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5316 00000100:00100000:0.0:1767320597.724033:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985466880:12345-0@lo:400:kworker.0 Request processed in 36us (164us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320597.724040:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3458 00000100:00080000:1.0:1767320597.724044:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320597.724050:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320597.724053:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320597.724086:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985466752:0@lo:400:kworker.0 00000100:00100000:1.0:1767320597.724093:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985466880:0@lo:400:kworker.0 00000100:00100000:0.0:1767320597.724104:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320597.724109:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985466240:0@lo:400:kworker.0 00000100:00100000:0.0:1767320597.724114:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985466496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320597.724138:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320597.724161:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985466496 00000100:00100000:1.0:1767320597.724195:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985466496 00000100:00100000:1.0:1767320597.724238:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5317 00000100:00100000:1.0:1767320597.724247:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985466496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320597.724296:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985466496:12345-0@lo:400:kworker.0 Request processed in 51us (157us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320597.724302:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5317 00000100:00100000:2.0:1767320597.724361:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985465984:0@lo:38:kworker.0 00000100:00100000:2.0:1767320597.724389:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320597.724419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985466112:0@lo:38:kworker.0 00000100:00100000:2.0:1767320597.724440:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320597.724458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985466368:0@lo:38:kworker.0 00000100:00100000:2.0:1767320597.724482:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320597.724495:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985466624:0@lo:38:kworker.0 00000100:00100000:1.0:1767320597.724547:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985465984 00000100:00100000:1.0:1767320597.724552:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985465984 00000100:00100000:0.0:1767320597.724562:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985466112 00000100:00100000:0.0:1767320597.724567:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985466112 00000100:00100000:1.0:1767320597.724574:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985466368 00000100:00100000:1.0:1767320597.724579:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985466368 00000100:00100000:0.0:1767320597.724595:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1997 00000100:00100000:0.0:1767320597.724602:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985466112:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320597.724608:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6026 00000100:00100000:1.0:1767320597.724613:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985465984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320597.724621:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320597.724625:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320597.724758:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985466496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320597.724782:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6027 00000100:00100000:3.0:1767320597.724787:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985466368:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320597.724797:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320597.727185:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320597.727197:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985466112:12345-0@lo:38:kworker.0 Request processed in 2591us (2752us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320597.727214:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1997 00000100:00080000:2.0:1767320597.727249:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320597.727256:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320597.727260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985466112:0@lo:38:kworker.0 00000100:00100000:0.0:1767320597.727279:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985466624 00000100:00100000:0.0:1767320597.727288:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985466624 00000100:00100000:0.0:1767320597.727330:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6028 00000100:00100000:0.0:1767320597.727336:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985466624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320597.727611:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320597.727658:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985466624:12345-0@lo:38:kworker.0 Request processed in 322us (503us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320597.727678:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6028 00000100:00100000:1.0:1767320597.729704:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985465984:12345-0@lo:38:kworker.0 Request processed in 5082us (5307us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320597.729717:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6026 00000100:00100000:3.0:1767320597.729768:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985466368:12345-0@lo:38:kworker.0 Request processed in 4977us (5283us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320597.729777:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6027 00000100:00080000:0.0:1767320597.732270:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320597.734784:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320597.734796:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320597.734801:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985465984:0@lo:38:kworker.0 00000100:00080000:3.0:1767320597.734814:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320597.734817:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320597.734820:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985466368:0@lo:38:kworker.0 00000100:00080000:3.0:1767320597.734830:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320597.734832:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320597.734835:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985466624:0@lo:38:kworker.0 00010000:00010000:1.0:1767320598.367048:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320599.387676:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320600.283743:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320600.283770:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985467008:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320600.283853:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320600.284234:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467008 00000100:00100000:0.0:1767320600.284244:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467008 00000100:00100000:0.0:1767320600.284314:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2658 00000100:00100000:0.0:1767320600.284324:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985467008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320600.284367:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320600.284436:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985467008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (582us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320600.284445:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2658 00000100:00080000:3.0:1767320600.284484:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320600.284903:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985467008:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320600.285129:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985467136:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320600.285219:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320600.289811:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467136 00000100:00100000:2.0:1767320600.289833:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467136 00000100:00100000:2.0:1767320600.290443:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2659 00000100:00100000:2.0:1767320600.290459:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985467136:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320600.290509:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320600.290667:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985467136:12345-0@lo:13:osp-pre-0-1.0 Request processed in 213us (5447us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320600.290688:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2659 00000100:00100000:0.0:1767320600.290909:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985467136:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320600.298557:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320600.417030:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320601.436495:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320601.775420:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320601.775822:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920626176 00000100:00100000:3.0:1767320601.775836:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920626176 00000100:00100000:0.0:1767320601.776064:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320601.776804:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920626304 00000100:00100000:3.0:1767320601.776810:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920626304 00000100:00100000:0.0:1767320601.776944:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6029 00000100:00100000:0.0:1767320601.776954:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920626176:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320601.776978:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320601.777057:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920626176:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 103us (1648us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320601.777068:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6029 00000100:00100000:3.0:1767320601.777595:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5318 00000100:00100000:3.0:1767320601.777605:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920626304:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320601.777837:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920626304:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 232us (1772us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320601.777844:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5318 00000100:00080000:2.0:1767320601.789861:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320602.459398:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320602.845655:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320602.845668:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320602.845671:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320602.845676:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320602.845680:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320602.845689:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5845 00000100:00080000:3.0:1767320602.845693:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320602.845700:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320602.845741:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05880 x1853163985467264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320602.845774:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320602.845778:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320602.845780:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320602.845784:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5845 00000100:00080000:3.0:1767320602.845786:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320602.845789:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320602.845802:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04a80 x1853163985467392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320602.845852:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320602.845855:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320602.845857:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320602.845861:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5845 00000100:00080000:3.0:1767320602.845863:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320602.845866:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320602.845879:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06d80 x1853163985467648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320602.845905:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320602.845909:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320602.845911:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320602.845914:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5845 00000100:00080000:3.0:1767320602.845916:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320602.845919:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320602.845930:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985467904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320602.845969:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985468032:0@lo:400:kworker.0 00000100:00100000:3.0:1767320602.846034:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320602.846099:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320602.846105:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985467776:0@lo:400:kworker.0 00000100:00100000:3.0:1767320602.846129:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320602.846188:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468032 00000100:00100000:3.0:1767320602.846195:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468032 00000100:00100000:3.0:1767320602.846256:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3459 00000100:00100000:3.0:1767320602.846265:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985468032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320602.846313:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985468032:12345-0@lo:400:kworker.0 Request processed in 49us (279us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320602.846321:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3459 00000100:00100000:3.0:1767320602.846342:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467776 00000100:00100000:3.0:1767320602.846345:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467776 00000100:00100000:3.0:1767320602.846372:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5319 00000100:00100000:3.0:1767320602.846378:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985467776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320602.846408:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985467776:12345-0@lo:400:kworker.0 Request processed in 31us (278us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320602.846415:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5319 00000100:00100000:3.0:1767320602.846468:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985467776:0@lo:400:kworker.0 00000100:00100000:3.0:1767320602.846478:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985468032:0@lo:400:kworker.0 00000100:00100000:2.0:1767320602.846562:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985467264:0@lo:38:kworker.0 00000100:00100000:2.0:1767320602.846593:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320602.846620:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985467392:0@lo:38:kworker.0 00000100:00100000:2.0:1767320602.846640:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320602.846657:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985467648:0@lo:38:kworker.0 00000100:00100000:2.0:1767320602.846673:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320602.846686:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985467904:0@lo:38:kworker.0 00000100:00100000:2.0:1767320602.846701:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320602.846728:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467264 00000100:00100000:2.0:1767320602.846731:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467264 00000100:00100000:2.0:1767320602.846748:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467648 00000100:00100000:2.0:1767320602.846751:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467648 00000100:00100000:2.0:1767320602.846763:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467904 00000100:00100000:2.0:1767320602.846765:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467904 00000100:00100000:2.0:1767320602.846775:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6030 00000100:00100000:2.0:1767320602.846780:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985467264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320602.846795:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320602.846826:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985467264:12345-0@lo:38:kworker.0 Request processed in 45us (232us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320602.846832:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6030 00000100:00100000:2.0:1767320602.846845:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6031 00000100:00100000:2.0:1767320602.846849:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985467648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320602.846858:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320602.846879:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985467648:12345-0@lo:38:kworker.0 Request processed in 29us (205us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320602.846884:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6031 00000100:00100000:2.0:1767320602.846895:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6032 00000100:00100000:2.0:1767320602.846899:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985467904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320602.846908:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320602.846929:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985467904:12345-0@lo:38:kworker.0 Request processed in 28us (226us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320602.846934:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6032 00000100:00080000:2.0:1767320602.846953:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320602.846957:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320602.846973:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320602.846978:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320602.846981:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985467264:0@lo:38:kworker.0 00000100:00080000:2.0:1767320602.846989:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320602.846992:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320602.846994:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985467648:0@lo:38:kworker.0 00000100:00080000:2.0:1767320602.847001:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320602.847003:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320602.847006:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985467904:0@lo:38:kworker.0 00000100:00100000:1.0:1767320602.847205:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985468160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320602.847249:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320602.847274:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320602.847279:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985467520:0@lo:400:kworker.0 00000100:00100000:1.0:1767320602.847299:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320602.847321:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468160 00000100:00100000:1.0:1767320602.847324:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468160 00000100:00100000:1.0:1767320602.847353:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3460 00000100:00100000:1.0:1767320602.847359:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985468160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320602.847390:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985468160:12345-0@lo:400:kworker.0 Request processed in 30us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320602.847395:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3460 00000100:00100000:1.0:1767320602.847420:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985468160:0@lo:400:kworker.0 00000100:00080000:1.0:1767320602.847431:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320602.847803:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467392 00000100:00100000:0.0:1767320602.847808:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467392 00000100:00100000:0.0:1767320602.847841:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1998 00000100:00100000:0.0:1767320602.847854:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985467392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320602.847867:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320602.847940:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985467392:12345-0@lo:38:kworker.0 Request processed in 85us (1298us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320602.847945:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1998 00000100:00100000:0.0:1767320602.847965:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985467520 00000100:00100000:0.0:1767320602.847968:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985467520 00000100:00100000:0.0:1767320602.847996:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5320 00000100:00100000:0.0:1767320602.848002:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985467520:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320602.848036:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985467520:12345-0@lo:400:kworker.0 Request processed in 34us (736us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320602.848042:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5320 00000100:00100000:0.0:1767320602.848073:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985467520:0@lo:400:kworker.0 00000100:00080000:0.0:1767320602.848087:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320602.848099:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320602.848103:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320602.848106:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985467392:0@lo:38:kworker.0 00010000:00010000:1.0:1767320603.483582:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320604.510811:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320605.404038:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320605.404062:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985468288:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320605.404138:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320605.404383:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468288 00000100:00100000:2.0:1767320605.404392:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468288 00000100:00100000:2.0:1767320605.404457:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2660 00000100:00100000:2.0:1767320605.404467:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985468288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320605.404496:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320605.404545:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985468288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (406us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320605.404552:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2660 00000100:00100000:2.0:1767320605.404599:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985468288:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320605.404856:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320605.409131:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320605.409155:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985468416:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320605.409266:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320605.409391:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468416 00000100:00100000:1.0:1767320605.409400:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468416 00000100:00100000:1.0:1767320605.409467:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2661 00000100:00100000:1.0:1767320605.409477:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985468416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320605.409508:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320605.409555:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985468416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (289us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320605.409563:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2661 00000100:00080000:3.0:1767320605.409629:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320605.409666:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985468416:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320605.531859:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320606.555553:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320606.892002:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320606.892041:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320606.892132:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920626560 00000100:00100000:2.0:1767320606.892212:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920626560 00000100:00100000:2.0:1767320606.892278:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920626432 00000100:00100000:2.0:1767320606.892282:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920626432 00000100:00100000:2.0:1767320606.892301:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5321 00000100:00100000:2.0:1767320606.892311:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920626560:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767320606.892725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320606.892769:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6033 00000100:00100000:1.0:1767320606.892774:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920626432:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320606.892796:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320606.892864:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920626432:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 88us (821us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320606.892870:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6033 00000100:00100000:2.0:1767320606.893491:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920626560:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 1180us (1496us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320606.893502:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5321 00010000:00010000:2.0:1767320607.579695:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320607.970235:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320607.970247:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320607.970250:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320607.970255:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320607.970260:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320607.970268:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5851 00000100:00080000:3.0:1767320607.970273:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320607.970281:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320607.970325:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc000 x1853163985468544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320607.970359:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320607.970363:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320607.970365:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320607.970368:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5851 00000100:00080000:3.0:1767320607.970371:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320607.970376:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320607.970389:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04a80 x1853163985468672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320607.970447:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320607.970452:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320607.970453:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320607.970457:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5851 00000100:00080000:3.0:1767320607.970460:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320607.970463:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320607.970474:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985468928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320607.970507:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320607.970510:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320607.970512:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320607.970516:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5851 00000100:00080000:3.0:1767320607.970518:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320607.970521:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320607.970531:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06d80 x1853163985469184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320607.970583:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985469056:0@lo:400:kworker.0 00000100:00100000:3.0:1767320607.970653:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320607.970729:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320607.970735:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985469312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320607.970797:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320607.970833:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469056 00000100:00100000:3.0:1767320607.970841:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469056 00000100:00100000:3.0:1767320607.970898:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5322 00000100:00100000:3.0:1767320607.970907:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985469056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320607.970960:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985469056:12345-0@lo:400:kworker.0 Request processed in 54us (306us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320607.970968:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5322 00000100:00100000:3.0:1767320607.971012:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985469056:0@lo:400:kworker.0 00000100:00100000:0.0:1767320607.971271:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985468800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320607.971302:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320607.971333:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320607.971339:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985469440:0@lo:400:kworker.0 00000100:00100000:0.0:1767320607.971360:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320607.971537:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468800 00000100:00100000:3.0:1767320607.971542:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468800 00000100:00100000:2.0:1767320607.971640:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469312 00000100:00100000:2.0:1767320607.971645:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469312 00000100:00100000:2.0:1767320607.971670:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469440 00000100:00100000:2.0:1767320607.971673:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469440 00000100:00100000:2.0:1767320607.971694:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3461 00000100:00100000:2.0:1767320607.971701:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985469312:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320607.971741:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985469312:12345-0@lo:400:kworker.0 Request processed in 41us (944us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320607.971748:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3461 00000100:00100000:2.0:1767320607.971761:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3462 00000100:00100000:2.0:1767320607.971766:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985469440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320607.971793:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985469440:12345-0@lo:400:kworker.0 Request processed in 28us (433us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320607.971798:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3462 00000100:00100000:3.0:1767320607.971836:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5323 00000100:00100000:3.0:1767320607.971843:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985468800:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320607.971871:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985468800:12345-0@lo:400:kworker.0 Request processed in 30us (569us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320607.971877:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5323 00000100:00080000:1.0:1767320607.971969:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320607.971974:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320607.971976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320607.972016:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985469440:0@lo:400:kworker.0 00000100:00100000:1.0:1767320607.972044:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985468800:0@lo:400:kworker.0 00000100:00100000:1.0:1767320607.972085:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985468544:0@lo:38:kworker.0 00000100:00100000:1.0:1767320607.972114:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320607.983581:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985468672:0@lo:38:kworker.0 00000100:00100000:1.0:1767320607.983650:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320607.983675:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985468928:0@lo:38:kworker.0 00000100:00100000:1.0:1767320607.983698:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320607.983757:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468544 00000100:00100000:0.0:1767320607.983767:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468544 00000100:00100000:0.0:1767320607.983817:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6034 00000100:00100000:0.0:1767320607.983824:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985468544:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320607.983844:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320607.983888:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985468544:12345-0@lo:38:kworker.0 Request processed in 63us (11773us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320607.983896:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6034 00000100:00100000:0.0:1767320607.983964:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468672 00000100:00100000:0.0:1767320607.983968:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468672 00000100:00100000:0.0:1767320607.983986:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1999 00000100:00100000:0.0:1767320607.983991:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985468672:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320607.984002:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320607.984025:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985468672:12345-0@lo:38:kworker.0 Request processed in 33us (375us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320607.984031:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1999 00000100:00100000:0.0:1767320607.984092:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985469312:0@lo:400:kworker.0 00000100:00100000:1.0:1767320607.990744:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985469184:0@lo:38:kworker.0 00000100:00100000:1.0:1767320607.990832:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320607.990908:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320607.990917:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320607.990921:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985468544:0@lo:38:kworker.0 00000100:00080000:1.0:1767320607.990935:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320607.990938:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320607.990941:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985468672:0@lo:38:kworker.0 00000100:00100000:3.0:1767320607.991111:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985468928 00000100:00100000:3.0:1767320607.991127:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985468928 00000100:00100000:3.0:1767320607.991204:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469184 00000100:00100000:3.0:1767320607.991208:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469184 00000100:00100000:3.0:1767320607.991235:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6035 00000100:00100000:3.0:1767320607.991243:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985468928:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320607.991272:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320607.991274:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6036 00000100:00100000:0.0:1767320607.991280:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985469184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320607.991288:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320607.991328:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985469184:12345-0@lo:38:kworker.0 Request processed in 45us (494us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320607.991328:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985468928:12345-0@lo:38:kworker.0 Request processed in 83us (7626us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320607.991335:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6036 00000100:00080000:1.0:1767320607.991802:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320607.991808:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320607.991812:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985468928:0@lo:38:kworker.0 00000100:00080000:1.0:1767320607.991821:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320607.991824:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320607.991827:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985469184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320607.991931:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6035 00010000:00010000:2.0:1767320608.608182:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320609.628014:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320610.523766:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320610.523783:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985469568:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320610.523863:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320610.525423:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469568 00000100:00100000:1.0:1767320610.525434:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469568 00000100:00100000:1.0:1767320610.525511:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2662 00000100:00100000:1.0:1767320610.525521:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985469568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320610.525551:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320610.525615:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985469568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (1752us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320610.525623:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2662 00000100:00080000:0.0:1767320610.525813:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320610.526842:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985469696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320610.526901:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320610.527011:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985469568:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320610.527300:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469696 00000100:00100000:2.0:1767320610.527308:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469696 00000100:00100000:2.0:1767320610.527379:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2663 00000100:00100000:2.0:1767320610.527395:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985469696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320610.527421:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320610.527468:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985469696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (566us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320610.527475:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2663 00000100:00100000:3.0:1767320610.527692:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985469696:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320610.527761:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320610.657446:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320611.675994:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320612.014323:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320612.014412:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320612.029225:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920626816 00000100:00100000:3.0:1767320612.029241:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920626816 00000100:00100000:3.0:1767320612.029409:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920626688 00000100:00100000:3.0:1767320612.029413:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920626688 00000100:00100000:1.0:1767320612.029516:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5324 00000100:00100000:1.0:1767320612.029530:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920626816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320612.029640:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920626816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 114us (15328us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320612.029648:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5324 00000100:00100000:1.0:1767320612.029670:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6037 00000100:00100000:1.0:1767320612.029674:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920626688:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320612.029694:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320612.029718:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920626688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (15306us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320612.029726:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6037 00000100:00080000:0.0:1767320612.029747:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320612.700932:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320613.083820:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320613.083831:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320613.083834:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320613.083838:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320613.083842:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320613.083850:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5856 00000100:00080000:3.0:1767320613.083854:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320613.083863:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320613.083900:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9d880 x1853163985469824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320613.083936:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320613.083939:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320613.083941:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320613.083944:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5856 00000100:00080000:3.0:1767320613.083947:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320613.083951:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320613.083966:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9c380 x1853163985469952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320613.083998:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320613.084002:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320613.084004:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320613.084008:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5856 00000100:00080000:3.0:1767320613.084011:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320613.084014:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320613.084024:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9e300 x1853163985470208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320613.084059:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320613.084063:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320613.084065:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320613.084068:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5856 00000100:00080000:3.0:1767320613.084071:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320613.084074:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320613.084083:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9ea00 x1853163985470464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320613.084124:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985470080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320613.084304:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320613.084366:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320613.084374:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985470720:0@lo:400:kworker.0 00000100:00100000:3.0:1767320613.084397:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320613.084445:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470080 00000100:00100000:3.0:1767320613.084454:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470080 00000100:00100000:3.0:1767320613.084502:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5325 00000100:00100000:3.0:1767320613.084510:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985470080:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320613.084708:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985470080:12345-0@lo:400:kworker.0 Request processed in 198us (404us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320613.084717:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5325 00000100:00100000:2.0:1767320613.084729:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985469824:0@lo:38:kworker.0 00000100:00100000:3.0:1767320613.084745:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985470336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320613.084762:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320613.084794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985469952:0@lo:38:kworker.0 00000100:00100000:2.0:1767320613.084818:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320613.084838:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985470208:0@lo:38:kworker.0 00000100:00100000:2.0:1767320613.084857:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320613.084870:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985470464:0@lo:38:kworker.0 00000100:00100000:0.0:1767320613.084884:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469952 00000100:00100000:2.0:1767320613.084888:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320613.084889:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469952 00000100:00100000:0.0:1767320613.084911:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2000 00000100:00100000:0.0:1767320613.084916:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985469952:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320613.084917:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985469824 00000100:00100000:2.0:1767320613.084919:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985469824 00010000:00020000:0.0:1767320613.084936:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320613.084936:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470208 00000100:00100000:2.0:1767320613.084938:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470208 00000100:00100000:2.0:1767320613.084949:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470464 00000100:00100000:2.0:1767320613.084951:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470464 00000100:00100000:2.0:1767320613.084959:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6038 00000100:00100000:2.0:1767320613.084964:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985469824:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320613.084972:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985469952:12345-0@lo:38:kworker.0 Request processed in 56us (153us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767320613.084974:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320613.084978:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2000 00000100:00100000:2.0:1767320613.084996:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985469824:12345-0@lo:38:kworker.0 Request processed in 30us (233us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320613.084998:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6039 00000100:00100000:0.0:1767320613.085002:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985470208:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320613.085002:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6038 00010000:00020000:0.0:1767320613.085011:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320613.085013:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6040 00000100:00100000:2.0:1767320613.085017:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985470464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320613.085025:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320613.085045:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985470208:12345-0@lo:38:kworker.0 Request processed in 41us (187us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320613.085052:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6039 00000100:00100000:2.0:1767320613.085056:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985470464:12345-0@lo:38:kworker.0 Request processed in 38us (167us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320613.085061:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6040 00000100:00080000:0.0:1767320613.085073:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320613.085097:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320613.085103:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320613.085106:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985469824:0@lo:38:kworker.0 00000100:00100000:2.0:1767320613.085107:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985470080:0@lo:400:kworker.0 00000100:00080000:0.0:1767320613.085114:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320613.085117:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320613.085119:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985469952:0@lo:38:kworker.0 00000100:00080000:0.0:1767320613.085128:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320613.085131:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320613.085134:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985470208:0@lo:38:kworker.0 00000100:00080000:0.0:1767320613.085140:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320613.085143:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320613.085145:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985470464:0@lo:38:kworker.0 00000100:00100000:1.0:1767320613.086410:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985470592:0@lo:400:kworker.0 00000100:00100000:1.0:1767320613.086467:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320613.086529:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470720 00000100:00100000:1.0:1767320613.086536:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470720 00000100:00100000:1.0:1767320613.086597:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470592 00000100:00100000:1.0:1767320613.086601:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470592 00000100:00080000:0.0:1767320613.086869:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320613.086873:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320613.086920:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3463 00000100:00100000:1.0:1767320613.086929:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985470720:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320613.086992:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985470720:12345-0@lo:400:kworker.0 Request processed in 64us (2593us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320613.087000:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3463 00000100:00100000:1.0:1767320613.087015:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3464 00000100:00100000:1.0:1767320613.087021:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985470592:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320613.087065:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985470592:12345-0@lo:400:kworker.0 Request processed in 45us (598us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320613.087071:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3464 00000100:00100000:3.0:1767320613.088833:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320613.088923:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470336 00000100:00100000:3.0:1767320613.088931:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470336 00000100:00100000:3.0:1767320613.088980:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5326 00000100:00100000:3.0:1767320613.088988:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985470336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320613.089048:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985470336:12345-0@lo:400:kworker.0 Request processed in 60us (216us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320613.089055:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5326 00000100:00080000:3.0:1767320613.089076:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320613.089110:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985470336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320613.091881:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985470720:0@lo:400:kworker.0 00000100:00100000:2.0:1767320613.091942:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985470592:0@lo:400:kworker.0 00010000:00010000:2.0:1767320613.724439:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320614.747681:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320615.643529:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985470976:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320615.643602:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320615.643646:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470976 00000100:00100000:2.0:1767320615.643655:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470976 00000100:00080000:3.0:1767320615.643688:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320615.643708:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2664 00000100:00100000:2.0:1767320615.643718:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985470976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320615.643750:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320615.643818:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320615.643845:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985470976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 128us (241us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320615.643853:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2664 00000100:00100000:1.0:1767320615.643907:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985470848:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320615.643923:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985470976:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320615.643936:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320615.643977:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985470848 00000100:00100000:2.0:1767320615.644018:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985470848 00000100:00100000:2.0:1767320615.644105:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2665 00000100:00100000:2.0:1767320615.644114:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985470848:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320615.644135:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320615.644207:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985470848:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320615.644214:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2665 00000100:00100000:0.0:1767320615.644356:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985470848:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320615.644582:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320615.776899:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320616.795423:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320617.130710:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320617.130815:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627200 00000100:00100000:3.0:1767320617.130825:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627200 00000100:00100000:3.0:1767320617.130916:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5327 00000100:00100000:3.0:1767320617.130926:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920627200:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320617.131026:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920627200:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (327us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320617.131034:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5327 00000100:00080000:0.0:1767320617.131152:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767320617.132685:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920626944 offset 0 length 520: 4 00000100:00100000:0.0:1767320617.132728:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320617.132818:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627072 00000100:00100000:1.0:1767320617.132827:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627072 00000100:00100000:1.0:1767320617.132917:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6041 00000100:00100000:1.0:1767320617.132925:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920627072:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320617.132968:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320617.133033:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920627072:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 107us (304us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320617.133040:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6041 00000100:00100000:0.0:1767320617.179886:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320592/real 1767320592] req@ffff8ce1a4dfa680 x1853163985465600/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320617 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320617.180159:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a4dfa680 x1853163985465600/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320617 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320617.180200:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320617.180227:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320617.180231:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985465600:0@lo:250:kworker.0 00000100:00080000:3.0:1767320617.181071:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320617.181079:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320617.181082:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320617.181088:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5840 00000100:00080000:3.0:1767320617.181093:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320617.181099:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320617.181129:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9c700 x1853163985471104/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320617.181159:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320617.181198:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320617.181200:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320617.181204:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5861 00000100:00080000:3.0:1767320617.181206:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320617.181210:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320617.181224:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9d180 x1853163985471232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320617.181232:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320617.181235:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320617.181236:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320617.181249:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5861 00000100:00080000:3.0:1767320617.181252:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320617.181255:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320617.181266:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9f480 x1853163985471360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320617.181304:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320617.181308:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320617.181309:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320617.181312:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5861 00000100:00080000:3.0:1767320617.181314:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320617.181318:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320617.181330:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9df80 x1853163985471616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320617.181351:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320617.181353:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320617.181355:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320617.181358:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5861 00000100:00080000:3.0:1767320617.181360:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320617.181362:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320617.181370:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9ea00 x1853163985471872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320617.181646:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985471488:0@lo:400:kworker.0 00000100:00100000:2.0:1767320617.181718:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320617.181803:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320617.181810:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985472128:0@lo:400:kworker.0 00000100:00100000:2.0:1767320617.181858:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320617.182225:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472128 00000100:00100000:3.0:1767320617.182232:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472128 00000100:00100000:3.0:1767320617.182289:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3465 00000100:00100000:3.0:1767320617.182297:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985472128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320617.182344:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985472128:12345-0@lo:400:kworker.0 Request processed in 47us (485us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320617.182353:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3465 00000100:00100000:3.0:1767320617.182377:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985471488 00000100:00100000:3.0:1767320617.182380:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985471488 00000100:00100000:3.0:1767320617.182405:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5328 00000100:00100000:3.0:1767320617.182410:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985471488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320617.182436:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985471488:12345-0@lo:400:kworker.0 Request processed in 26us (719us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320617.182442:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5328 00000100:00080000:2.0:1767320617.182607:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320617.182611:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320617.182640:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985472128:0@lo:400:kworker.0 00000100:00100000:2.0:1767320617.182649:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985471488:0@lo:400:kworker.0 00000100:00100000:0.0:1767320617.182728:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985471744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320617.182758:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320617.182799:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320617.182805:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985472000:0@lo:400:kworker.0 00000100:00100000:0.0:1767320617.182826:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320617.182864:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985471104:0@lo:250:kworker.0 00000400:00000100:0.0:1767320617.182884:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985471104 offset 0 length 520: 4 00000100:00100000:0.0:1767320617.182900:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985471232:0@lo:38:kworker.0 00000100:00100000:0.0:1767320617.182920:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320617.182934:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985471360:0@lo:38:kworker.0 00000100:00100000:2.0:1767320617.183208:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472000 00000100:00100000:2.0:1767320617.183212:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472000 00000100:00100000:2.0:1767320617.183254:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3466 00000100:00100000:2.0:1767320617.183263:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985472000:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320617.183300:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985472000:12345-0@lo:400:kworker.0 Request processed in 38us (472us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320617.183307:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3466 00000100:00080000:2.0:1767320617.183322:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320617.183339:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985472000:0@lo:400:kworker.0 00000100:00100000:0.0:1767320617.183487:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320617.183516:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985471616:0@lo:38:kworker.0 00000100:00100000:0.0:1767320617.183537:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320617.183552:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985471872:0@lo:38:kworker.0 00000100:00100000:0.0:1767320617.183570:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320617.183599:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985471744 00000100:00100000:0.0:1767320617.183603:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985471744 00000100:00100000:0.0:1767320617.183629:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985471232 00000100:00100000:0.0:1767320617.183632:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985471232 00000100:00100000:0.0:1767320617.183644:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985471616 00000100:00100000:0.0:1767320617.183646:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985471616 00000100:00100000:0.0:1767320617.183653:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985471872 00000100:00100000:0.0:1767320617.183655:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985471872 00000100:00100000:0.0:1767320617.183663:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5329 00000100:00100000:0.0:1767320617.183668:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985471744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320617.183697:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985471744:12345-0@lo:400:kworker.0 Request processed in 29us (939us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320617.183703:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5329 00000100:00100000:0.0:1767320617.183715:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6042 00000100:00100000:0.0:1767320617.183719:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985471232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320617.183736:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320617.183765:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985471232:12345-0@lo:38:kworker.0 Request processed in 44us (844us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320617.183928:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6043 00000100:00100000:1.0:1767320617.183935:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985471616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320617.183946:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320617.183992:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985471616:12345-0@lo:38:kworker.0 Request processed in 56us (453us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320617.183999:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6043 00000100:00100000:1.0:1767320617.184011:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6044 00000100:00100000:1.0:1767320617.184015:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985471872:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320617.184023:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320617.184055:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985471872:12345-0@lo:38:kworker.0 Request processed in 39us (484us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320617.184060:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6044 00000100:00080000:1.0:1767320617.184089:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320617.184095:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320617.184098:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985471232:0@lo:38:kworker.0 00000100:00080000:1.0:1767320617.184106:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320617.184110:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320617.184112:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985471616:0@lo:38:kworker.0 00000100:00080000:1.0:1767320617.184120:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320617.184123:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320617.184126:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985471872:0@lo:38:kworker.0 00000100:00100000:0.0:1767320617.184401:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6042 00000100:00100000:2.0:1767320617.184468:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985471360 00000100:00100000:2.0:1767320617.184473:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985471360 00000100:00100000:2.0:1767320617.184502:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2001 00000100:00100000:2.0:1767320617.184506:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985471360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320617.184516:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320617.184555:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985471360:12345-0@lo:38:kworker.0 Request processed in 47us (1068us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320617.184561:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2001 00000100:00080000:2.0:1767320617.184584:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320617.184605:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985471744:0@lo:400:kworker.0 00000100:00080000:1.0:1767320617.184985:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320617.184989:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320617.184992:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985471360:0@lo:38:kworker.0 00010000:00010000:1.0:1767320617.819439:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320618.843475:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320619.868300:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320620.764086:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985472256:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320620.764158:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320620.764340:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472256 00000100:00100000:2.0:1767320620.764349:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472256 00000100:00100000:2.0:1767320620.764415:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2666 00000100:00100000:2.0:1767320620.764424:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985472256:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320620.764451:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320620.764494:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985472256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (337us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320620.764501:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2666 00000100:00100000:2.0:1767320620.764547:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985472256:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320620.764786:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320620.764796:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985472384:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320620.764823:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320620.764848:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472384 00000100:00100000:3.0:1767320620.764851:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472384 00000100:00100000:3.0:1767320620.764875:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2667 00000100:00100000:3.0:1767320620.764881:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985472384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320620.764908:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320620.764937:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985472384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (113us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320620.764943:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2667 00000100:00100000:3.0:1767320620.764969:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985472384:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320620.765074:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320620.765078:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320620.898791:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320621.915374:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320622.250611:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320622.250656:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320622.250741:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627456 00000100:00100000:3.0:1767320622.250751:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627456 00000100:00100000:3.0:1767320622.250826:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627328 00000100:00100000:3.0:1767320622.250829:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627328 00000100:00100000:3.0:1767320622.250847:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5330 00000100:00100000:3.0:1767320622.250857:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920627456:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320622.250966:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920627456:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 111us (366us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320622.250972:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5330 00000100:00100000:3.0:1767320622.250990:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6045 00000100:00100000:3.0:1767320622.250994:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920627328:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320622.251012:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320622.251038:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920627328:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (382us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320622.251045:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6045 00000100:00080000:2.0:1767320622.251357:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320622.299769:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.299780:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320622.299784:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.299788:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.299792:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.299801:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.299806:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.299813:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320622.299887:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552fb80 x1853163985472512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320622.299912:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.299916:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.299918:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.299922:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.299924:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.299929:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320622.299945:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552dc00 x1853163985472640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320622.300004:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.300008:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.300009:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.300013:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.300015:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.300019:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320622.300041:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552d180 x1853163985472896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320622.300075:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.300079:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.300081:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.300084:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.300087:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.300090:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320622.300100:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552c000 x1853163985473152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320622.300147:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985473024:0@lo:400:kworker.0 00000100:00100000:0.0:1767320622.300239:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320622.300320:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320622.300327:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985473280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320622.300355:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320622.300390:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473024 00000100:00100000:0.0:1767320622.300397:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473024 00000100:00100000:3.0:1767320622.300560:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985472768:0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.300590:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320622.300610:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320622.300615:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985473408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.300636:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320622.300685:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473280 00000100:00100000:3.0:1767320622.300689:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473280 00000100:00100000:3.0:1767320622.300720:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473408 00000100:00100000:3.0:1767320622.300723:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473408 00000100:00100000:3.0:1767320622.300744:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3467 00000100:00100000:3.0:1767320622.300752:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985473280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.300799:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985473280:12345-0@lo:400:kworker.0 Request processed in 48us (443us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320622.300806:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3467 00000100:00100000:3.0:1767320622.300822:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3468 00000100:00100000:3.0:1767320622.300828:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985473408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.300912:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985473408:12345-0@lo:400:kworker.0 Request processed in 84us (274us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320622.300919:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3468 00000100:00100000:3.0:1767320622.300954:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985473408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.300990:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985473280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320622.301708:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472768 00000100:00100000:0.0:1767320622.301714:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472768 00000100:00100000:0.0:1767320622.301744:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5331 00000100:00100000:0.0:1767320622.301751:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985473024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320622.301795:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985473024:12345-0@lo:400:kworker.0 Request processed in 45us (1556us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320622.301802:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5331 00000100:00100000:0.0:1767320622.301815:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5332 00000100:00100000:0.0:1767320622.301822:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985472768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320622.301851:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985472768:12345-0@lo:400:kworker.0 Request processed in 30us (1261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320622.301869:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5332 00000100:00100000:0.0:1767320622.301902:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985472768:0@lo:400:kworker.0 00000100:00100000:0.0:1767320622.301923:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985473024:0@lo:400:kworker.0 00000100:00080000:2.0:1767320622.302020:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320622.302025:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320622.302028:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320622.302724:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985472512:0@lo:38:kworker.0 00000100:00100000:1.0:1767320622.302773:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320622.302807:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985472640:0@lo:38:kworker.0 00000100:00100000:1.0:1767320622.302833:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320622.302876:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985472896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320622.302900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320622.302919:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985473152:0@lo:38:kworker.0 00000100:00100000:1.0:1767320622.302938:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320622.302969:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472512 00000100:00100000:1.0:1767320622.302976:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472512 00000100:00100000:1.0:1767320622.303000:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472896 00000100:00100000:1.0:1767320622.303003:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472896 00000100:00100000:1.0:1767320622.303011:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473152 00000100:00100000:1.0:1767320622.303013:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473152 00000100:00100000:1.0:1767320622.303023:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6046 00000100:00100000:1.0:1767320622.303029:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985472512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320622.303050:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320622.303087:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985472512:12345-0@lo:38:kworker.0 Request processed in 57us (313us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320622.303094:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6046 00000100:00100000:1.0:1767320622.303109:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6047 00000100:00100000:1.0:1767320622.303114:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985472896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320622.303123:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320622.303143:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985472896:12345-0@lo:38:kworker.0 Request processed in 29us (243us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320622.303180:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6047 00000100:00100000:1.0:1767320622.303193:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6048 00000100:00100000:1.0:1767320622.303197:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985473152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320622.303206:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320622.303230:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985473152:12345-0@lo:38:kworker.0 Request processed in 32us (291us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320622.303236:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6048 00000100:00080000:1.0:1767320622.303276:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320622.303282:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320622.303286:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985472512:0@lo:38:kworker.0 00000100:00080000:1.0:1767320622.303295:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320622.303298:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320622.303301:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985472896:0@lo:38:kworker.0 00000100:00080000:1.0:1767320622.303309:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320622.303312:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320622.303315:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985473152:0@lo:38:kworker.0 00000100:00100000:2.0:1767320622.303486:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985472640 00000100:00100000:2.0:1767320622.303492:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985472640 00000100:00100000:2.0:1767320622.303527:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2002 00000100:00100000:2.0:1767320622.303533:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985472640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320622.303544:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320622.303582:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985472640:12345-0@lo:38:kworker.0 Request processed in 48us (748us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320622.303588:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2002 00000100:00080000:2.0:1767320622.303614:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320622.303618:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320622.303621:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985472640:0@lo:38:kworker.0 00010000:00010000:1.0:1767320622.939399:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320623.966978:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320624.989785:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320625.883461:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985473536:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320625.883532:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320625.883897:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473536 00000100:00100000:3.0:1767320625.883906:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473536 00000100:00100000:3.0:1767320625.883969:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2668 00000100:00100000:3.0:1767320625.883978:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985473536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320625.884020:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320625.884076:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985473536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (543us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320625.884084:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2668 00000100:00080000:2.0:1767320625.884222:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320625.884331:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985473536:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320625.884385:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320625.884395:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985473664:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320625.884425:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320625.884501:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473664 00000100:00100000:0.0:1767320625.884506:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473664 00000100:00100000:0.0:1767320625.884543:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2669 00000100:00100000:0.0:1767320625.884550:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985473664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320625.884566:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320625.884594:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985473664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 45us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320625.884600:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2669 00000100:00080000:2.0:1767320625.884747:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320625.884776:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985473664:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320626.011590:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320627.038741:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320627.379056:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320627.379113:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320627.379231:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627584 00000100:00100000:3.0:1767320627.379279:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627712 00000100:00100000:0.0:1767320627.379283:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627584 00000100:00100000:3.0:1767320627.379283:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627712 00000100:00100000:0.0:1767320627.379329:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5333 00000100:00100000:3.0:1767320627.379338:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6049 00000100:00100000:0.0:1767320627.379341:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920627712:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320627.379343:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920627584:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320627.379365:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320627.379412:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920627584:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 66us (365us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320627.379420:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6049 00000100:00080000:2.0:1767320627.379496:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320627.380087:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920627712:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 747us (970us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320627.380094:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5333 00000100:00080000:0.0:1767320627.419843:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.419854:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320627.419857:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.419861:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320627.419865:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.419872:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.419876:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320627.419883:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320627.420134:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552ca80 x1853163985473792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320627.420158:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.420177:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320627.420180:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.420184:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.420186:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320627.420190:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320627.420204:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552e300 x1853163985473920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320627.420235:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.420238:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767320627.420239:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985473792:0@lo:38:kworker.0 00000100:00080000:0.0:1767320627.420240:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.420243:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.420245:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320627.420248:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320627.420257:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552ea00 x1853163985474176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320627.420279:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.420283:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320627.420284:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.420287:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.420290:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320627.420292:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320627.420300:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552ce00 x1853163985474432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320627.420304:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320627.420332:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985474048:0@lo:400:kworker.0 00000100:00100000:3.0:1767320627.420353:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985473920:0@lo:38:kworker.0 00000100:00100000:0.0:1767320627.420357:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320627.420378:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320627.420380:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320627.420384:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985474688:0@lo:400:kworker.0 00000100:00100000:0.0:1767320627.420404:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320627.420404:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985474176:0@lo:38:kworker.0 00000100:00100000:3.0:1767320627.420421:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320627.420429:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985474304:0@lo:400:kworker.0 00000100:00100000:3.0:1767320627.420435:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985474432:0@lo:38:kworker.0 00000100:00100000:0.0:1767320627.420449:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320627.420460:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473920 00000100:00100000:3.0:1767320627.420466:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473920 00000100:00100000:0.0:1767320627.420468:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320627.420472:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985474560:0@lo:400:kworker.0 00000100:00100000:0.0:1767320627.420489:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320627.420498:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2003 00000100:00100000:3.0:1767320627.420505:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985473920:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320627.420510:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985473792 00000100:00100000:0.0:1767320627.420513:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985473792 00010000:00020000:3.0:1767320627.420524:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320627.420530:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474048 00000100:00100000:0.0:1767320627.420532:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474048 00000100:00100000:3.0:1767320627.420560:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985473920:12345-0@lo:38:kworker.0 Request processed in 55us (179us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320627.420562:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474176 00000100:00100000:0.0:1767320627.420564:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474176 00000100:00100000:3.0:1767320627.420568:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2003 00000100:00100000:0.0:1767320627.420571:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474304 00000100:00100000:0.0:1767320627.420573:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474304 00000100:00100000:0.0:1767320627.420587:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5334 00000100:00100000:3.0:1767320627.420590:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5335 00000100:00100000:0.0:1767320627.420592:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985474048:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320627.420596:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985474304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320627.420629:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985474048:12345-0@lo:400:kworker.0 Request processed in 36us (270us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320627.420629:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985474304:12345-0@lo:400:kworker.0 Request processed in 32us (178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320627.420635:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5334 00000100:00100000:2.0:1767320627.420771:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474688 00000100:00100000:2.0:1767320627.420776:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474688 00000100:00100000:2.0:1767320627.420803:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474560 00000100:00100000:2.0:1767320627.420805:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474560 00000100:00100000:2.0:1767320627.420825:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3469 00000100:00100000:2.0:1767320627.420833:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985474688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320627.420866:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985474688:12345-0@lo:400:kworker.0 Request processed in 35us (460us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320627.420872:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3469 00000100:00100000:2.0:1767320627.420885:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3470 00000100:00100000:2.0:1767320627.420891:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985474560:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320627.420917:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985474560:12345-0@lo:400:kworker.0 Request processed in 26us (426us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320627.420922:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3470 00000100:00080000:2.0:1767320627.420940:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320627.420943:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320627.420945:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320627.420978:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985474560:0@lo:400:kworker.0 00000100:00100000:2.0:1767320627.420989:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985474304:0@lo:400:kworker.0 00000100:00100000:3.0:1767320627.422598:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5335 00000100:00100000:3.0:1767320627.422632:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6050 00000100:00100000:3.0:1767320627.422638:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985473792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320627.422657:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320627.422712:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985473792:12345-0@lo:38:kworker.0 Request processed in 73us (2407us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320627.422718:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6050 00000100:00100000:3.0:1767320627.422731:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6051 00000100:00100000:3.0:1767320627.422735:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985474176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320627.422745:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320627.427461:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985474176:12345-0@lo:38:kworker.0 Request processed in 4714us (7028us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320627.427476:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6051 00000100:00100000:3.0:1767320627.427559:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320627.427607:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.427613:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.427617:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985473792:0@lo:38:kworker.0 00000100:00080000:3.0:1767320627.427627:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.427631:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.427633:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985473920:0@lo:38:kworker.0 00000100:00080000:3.0:1767320627.427656:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.427659:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.427662:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985474176:0@lo:38:kworker.0 00000100:00100000:3.0:1767320627.427681:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474432 00000100:00100000:3.0:1767320627.427687:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474432 00000100:00100000:3.0:1767320627.427714:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6052 00000100:00100000:3.0:1767320627.427720:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985474432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320627.427739:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320627.427796:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985474432:12345-0@lo:38:kworker.0 Request processed in 75us (237us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320627.427802:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6052 00000100:00080000:3.0:1767320627.427825:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.427828:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.427832:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985474432:0@lo:38:kworker.0 00000100:00100000:3.0:1767320627.427866:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985474688:0@lo:400:kworker.0 00000100:00100000:3.0:1767320627.427875:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985474048:0@lo:400:kworker.0 00010000:00010000:1.0:1767320628.060289:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320629.086757:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320630.107691:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320631.003458:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985474816:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320631.003532:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320631.003750:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474816 00000100:00100000:0.0:1767320631.003759:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474816 00000100:00100000:0.0:1767320631.003822:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2670 00000100:00100000:0.0:1767320631.003832:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985474816:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320631.003862:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320631.003908:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985474816:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (374us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320631.003915:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2670 00000100:00080000:1.0:1767320631.003983:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320631.004019:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985474816:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320631.017511:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320631.017547:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985474944:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320631.017638:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320631.023241:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985474944 00000100:00100000:1.0:1767320631.023259:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985474944 00000100:00100000:1.0:1767320631.023348:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2671 00000100:00100000:1.0:1767320631.023358:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985474944:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320631.023365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767320631.023390:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320631.023476:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985474944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (5837us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320631.023483:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2671 00000100:00100000:3.0:1767320631.023574:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985474944:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320631.131911:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320632.159586:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320632.490619:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320632.490738:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627840 00000100:00100000:2.0:1767320632.490750:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627840 00000100:00100000:2.0:1767320632.490808:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6053 00000100:00100000:2.0:1767320632.490817:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920627840:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:3.0:1767320632.490840:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00010000:00020000:2.0:1767320632.490844:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320632.490921:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920627840:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 104us (312us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320632.490929:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6053 00000100:00100000:2.0:1767320632.490944:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920627968 00000100:00100000:2.0:1767320632.490948:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920627968 00000100:00100000:2.0:1767320632.490982:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5336 00000100:00100000:2.0:1767320632.490990:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920627968:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320632.491040:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920627968:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 52us (202us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320632.491047:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5336 00000100:00080000:2.0:1767320632.491086:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320632.539347:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.539359:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320632.539362:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.539366:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.539370:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.539378:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.539382:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320632.539402:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320632.539442:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9c380 x1853163985475072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320632.539459:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.539462:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.539464:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.539467:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.539469:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320632.539472:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320632.539484:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9ce00 x1853163985475200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320632.539529:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.539533:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.539535:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.539538:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.539541:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320632.539544:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320632.539555:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06300 x1853163985475456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320632.539581:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.539584:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.539585:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.539588:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.539590:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320632.539593:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320632.539602:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05880 x1853163985475712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320632.539638:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985475584:0@lo:400:kworker.0 00000100:00100000:3.0:1767320632.539706:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320632.539767:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320632.539774:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985475840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320632.539821:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320632.539857:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475840 00000100:00100000:3.0:1767320632.539865:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475840 00000100:00100000:3.0:1767320632.539911:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3471 00000100:00100000:3.0:1767320632.539921:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985475840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320632.539969:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985475840:12345-0@lo:400:kworker.0 Request processed in 49us (146us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320632.539976:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3471 00000100:00100000:3.0:1767320632.540017:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985475840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320632.540045:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475584 00000100:00100000:3.0:1767320632.540049:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475584 00000100:00080000:3.0:1767320632.540070:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320632.540099:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5337 00000100:00100000:3.0:1767320632.540106:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985475584:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320632.540138:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985475584:12345-0@lo:400:kworker.0 Request processed in 33us (431us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320632.540144:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5337 00000100:00100000:3.0:1767320632.540187:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985475584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320632.540273:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985475072:0@lo:38:kworker.0 00000100:00100000:0.0:1767320632.540303:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320632.540332:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985475200:0@lo:38:kworker.0 00000100:00100000:0.0:1767320632.540356:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320632.540379:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985475456:0@lo:38:kworker.0 00000100:00100000:0.0:1767320632.540398:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320632.540411:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985475712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320632.540428:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320632.540458:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475200 00000100:00100000:0.0:1767320632.540462:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475200 00000100:00100000:0.0:1767320632.540482:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2004 00000100:00100000:0.0:1767320632.540487:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985475200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320632.540505:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320632.540537:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985475200:12345-0@lo:38:kworker.0 Request processed in 48us (180us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320632.540543:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2004 00000100:00100000:0.0:1767320632.540562:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475072 00000100:00100000:0.0:1767320632.540565:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475072 00000100:00100000:0.0:1767320632.540580:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475456 00000100:00100000:0.0:1767320632.540583:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475456 00000100:00100000:0.0:1767320632.540593:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475712 00000100:00100000:0.0:1767320632.540595:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475712 00000100:00100000:0.0:1767320632.540608:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6054 00000100:00100000:0.0:1767320632.540612:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985475072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320632.540622:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320632.540646:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985475072:12345-0@lo:38:kworker.0 Request processed in 32us (342us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320632.540655:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6054 00000100:00100000:0.0:1767320632.540666:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6055 00000100:00100000:0.0:1767320632.540683:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985475456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320632.540692:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320632.540717:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985475456:12345-0@lo:38:kworker.0 Request processed in 32us (317us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320632.540722:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6055 00000100:00100000:0.0:1767320632.540733:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6056 00000100:00100000:0.0:1767320632.540738:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985475712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320632.540746:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320632.540768:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985475712:12345-0@lo:38:kworker.0 Request processed in 29us (338us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320632.540773:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6056 00000100:00080000:0.0:1767320632.540797:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320632.540801:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320632.540804:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985475072:0@lo:38:kworker.0 00000100:00080000:0.0:1767320632.540812:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320632.540815:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320632.540818:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985475200:0@lo:38:kworker.0 00000100:00080000:0.0:1767320632.540824:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320632.540827:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320632.540830:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985475456:0@lo:38:kworker.0 00000100:00080000:0.0:1767320632.540836:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320632.540840:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320632.540842:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985475712:0@lo:38:kworker.0 00000100:00100000:1.0:1767320632.540936:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985475328:0@lo:400:kworker.0 00000100:00100000:1.0:1767320632.540966:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320632.540994:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320632.541000:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985475968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320632.541023:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320632.541082:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475328 00000100:00100000:0.0:1767320632.541086:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475328 00000100:00100000:0.0:1767320632.541124:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5338 00000100:00100000:0.0:1767320632.541130:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985475328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320632.541185:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985475328:12345-0@lo:400:kworker.0 Request processed in 34us (197us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320632.541191:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5338 00000100:00080000:1.0:1767320632.541309:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320632.541332:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985475328:0@lo:400:kworker.0 00000100:00100000:3.0:1767320632.541631:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985475968 00000100:00100000:3.0:1767320632.541636:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985475968 00000100:00100000:3.0:1767320632.541681:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3472 00000100:00100000:3.0:1767320632.541688:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985475968:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320632.541732:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985475968:12345-0@lo:400:kworker.0 Request processed in 45us (709us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320632.541739:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3472 00000100:00100000:1.0:1767320632.545019:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985475968:0@lo:400:kworker.0 00010000:00010000:2.0:1767320633.179418:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320634.206132:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320635.227493:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320636.123632:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320636.123655:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985476096:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320636.123724:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320636.123798:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476096 00000100:00100000:2.0:1767320636.123807:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476096 00000100:00100000:2.0:1767320636.123880:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2672 00000100:00100000:2.0:1767320636.123891:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985476096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320636.123923:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320636.123982:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985476096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (257us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320636.123989:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2672 00000100:00100000:1.0:1767320636.124036:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985476096:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320636.124874:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985476224:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320636.124913:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320636.124992:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476224 00000100:00100000:2.0:1767320636.124998:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476224 00000100:00100000:2.0:1767320636.125031:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2673 00000100:00100000:2.0:1767320636.125040:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985476224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320636.125060:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320636.125107:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985476224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (192us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320636.125114:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2673 00000100:00100000:0.0:1767320636.125272:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985476224:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320636.126199:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320636.126206:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320636.251519:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320637.275512:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320637.611631:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320637.611671:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320637.611833:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920628096 00000100:00100000:1.0:1767320637.611843:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920628096 00000100:00100000:1.0:1767320637.611895:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920628224 00000100:00100000:1.0:1767320637.611899:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920628224 00000100:00100000:1.0:1767320637.611950:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5339 00000100:00100000:1.0:1767320637.611970:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920628224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320637.612041:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920628224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (368us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320637.612049:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5339 00000100:00100000:1.0:1767320637.612067:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6057 00000100:00100000:1.0:1767320637.612073:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920628096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320637.612095:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320637.612122:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920628096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (499us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320637.612128:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6057 00000100:00080000:0.0:1767320637.612131:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320637.662102:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320637.662112:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320637.662115:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320637.662119:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320637.662123:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320637.662131:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5880 00000100:00080000:1.0:1767320637.662136:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320637.662143:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320637.662208:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985476352/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320637.662244:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320637.662248:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320637.662250:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320637.662253:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5880 00000100:00080000:1.0:1767320637.662255:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320637.662259:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320637.662272:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3800 x1853163985476480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320637.662310:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320637.662314:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320637.662316:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320637.662320:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5880 00000100:00080000:1.0:1767320637.662322:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320637.662325:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320637.662363:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985476736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320637.662388:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320637.662401:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320637.662403:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320637.662406:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5880 00000100:00080000:1.0:1767320637.662409:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320637.662412:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320637.662425:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7480 x1853163985476992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320637.662470:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985476864:0@lo:400:kworker.0 00000100:00100000:1.0:1767320637.662530:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320637.662611:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320637.662618:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985477120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320637.662646:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320637.662753:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985476352:0@lo:38:kworker.0 00000100:00100000:0.0:1767320637.662782:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320637.662811:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985476480:0@lo:38:kworker.0 00000100:00100000:0.0:1767320637.662838:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320637.662862:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985476736:0@lo:38:kworker.0 00000100:00100000:0.0:1767320637.662880:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320637.662896:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985476992:0@lo:38:kworker.0 00000100:00100000:0.0:1767320637.662914:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320637.662945:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476480 00000100:00100000:0.0:1767320637.662951:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476480 00000100:00100000:0.0:1767320637.662984:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2005 00000100:00100000:0.0:1767320637.662990:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190dbb850 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985476480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320637.663011:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320637.663050:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190dbb850 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985476480:12345-0@lo:38:kworker.0 Request processed in 58us (210us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320637.663057:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2005 00000100:00100000:3.0:1767320637.663068:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985476608:0@lo:400:kworker.0 00000100:00100000:3.0:1767320637.663104:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320637.663132:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320637.663152:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985477248:0@lo:400:kworker.0 00000100:00100000:3.0:1767320637.663205:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320637.663241:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985477120 00000100:00100000:3.0:1767320637.663245:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985477120 00000100:00100000:0.0:1767320637.663269:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476864 00000100:00100000:0.0:1767320637.663275:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476864 00000100:00100000:3.0:1767320637.663283:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985477248 00000100:00100000:3.0:1767320637.663288:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985477248 00000100:00100000:3.0:1767320637.663317:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3473 00000100:00100000:0.0:1767320637.663322:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476352 00000100:00100000:3.0:1767320637.663326:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985477120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320637.663329:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476352 00000100:00100000:0.0:1767320637.663343:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476736 00000100:00100000:0.0:1767320637.663346:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476736 00000100:00100000:0.0:1767320637.663358:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476992 00000100:00100000:0.0:1767320637.663361:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476992 00000100:00100000:0.0:1767320637.663375:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985476608 00000100:00100000:0.0:1767320637.663378:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985476608 00000100:00100000:3.0:1767320637.663382:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985477120:12345-0@lo:400:kworker.0 Request processed in 56us (733us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320637.663389:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3473 00000100:00100000:0.0:1767320637.663401:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5340 00000100:00100000:3.0:1767320637.663410:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3474 00000100:00100000:0.0:1767320637.663412:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985476864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320637.663422:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985477248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320637.663457:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985476864:12345-0@lo:400:kworker.0 Request processed in 48us (925us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320637.663466:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5340 00000100:00100000:3.0:1767320637.663471:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985477248:12345-0@lo:400:kworker.0 Request processed in 52us (266us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320637.663479:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3474 00000100:00100000:0.0:1767320637.663484:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5341 00000100:00100000:0.0:1767320637.663493:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985476608:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320637.663528:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985477248:0@lo:400:kworker.0 00000100:00100000:2.0:1767320637.663621:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6058 00000100:00100000:2.0:1767320637.663627:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985476352:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320637.663638:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320637.664640:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6059 00000100:00100000:1.0:1767320637.664648:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985476736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320637.664663:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320637.664762:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985476608:12345-0@lo:400:kworker.0 Request processed in 1270us (1659us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320637.664770:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5341 00000100:00100000:0.0:1767320637.664792:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6060 00000100:00100000:0.0:1767320637.664796:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985476992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320637.664805:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320637.664830:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985476992:12345-0@lo:38:kworker.0 Request processed in 32us (1915us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320637.664836:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6060 00000100:00080000:0.0:1767320637.664863:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320637.664892:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320637.664897:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320637.664901:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985476480:0@lo:38:kworker.0 00000100:00080000:0.0:1767320637.664910:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320637.664913:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320637.664916:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985476992:0@lo:38:kworker.0 00000100:00100000:0.0:1767320637.665073:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985476608:0@lo:400:kworker.0 00000100:00100000:3.0:1767320637.665554:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985477120:0@lo:400:kworker.0 00000100:00100000:3.0:1767320637.665569:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985476864:0@lo:400:kworker.0 00000100:00100000:2.0:1767320637.666598:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985476352:12345-0@lo:38:kworker.0 Request processed in 2965us (3810us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320637.666610:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6058 00000100:00080000:0.0:1767320637.668417:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320637.668428:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320637.668433:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985476352:0@lo:38:kworker.0 00000100:00100000:1.0:1767320637.668861:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985476736:12345-0@lo:38:kworker.0 Request processed in 4208us (5975us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320637.668873:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6059 00000100:00080000:0.0:1767320637.669077:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320637.669082:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320637.669085:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985476736:0@lo:38:kworker.0 00010000:00010000:2.0:1767320638.299437:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320639.327627:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320640.347400:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320641.243397:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985477504:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320641.243444:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320641.243455:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985477376:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320641.243470:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320641.243611:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985477504 00000100:00100000:0.0:1767320641.243619:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985477504 00000100:00100000:1.0:1767320641.243634:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320641.243681:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985477376 00000100:00100000:0.0:1767320641.243685:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985477376 00000100:00100000:1.0:1767320641.243700:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2674 00000100:00080000:3.0:1767320641.243851:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320641.243856:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320641.243930:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985477504:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320641.243978:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320641.244028:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985477504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (558us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320641.244099:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985477504:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320641.247663:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2675 00000100:00100000:0.0:1767320641.247688:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985477376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320641.247722:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320641.247826:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985477376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 151us (4193us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320641.247836:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2675 00000100:00100000:0.0:1767320641.247927:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985477376:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320641.248296:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2674 00010000:00010000:1.0:1767320641.372578:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320641.707900:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767320641.707945:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920628352 offset 0 length 520: 4 00000100:00100000:0.0:1767320641.707970:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320641.708058:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920628608 00000100:00100000:1.0:1767320641.708069:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920628608 00000100:00100000:1.0:1767320641.708141:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920628480 00000100:00100000:1.0:1767320641.708144:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920628480 00000100:00100000:1.0:1767320641.708196:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5342 00000100:00100000:1.0:1767320641.708207:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920628608:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320641.708284:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920628608:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320641.708292:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5342 00000100:00100000:1.0:1767320641.708310:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6061 00000100:00100000:1.0:1767320641.708314:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920628480:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320641.708335:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320641.708364:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920628480:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (394us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320641.708371:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6061 00000100:00080000:3.0:1767320641.708533:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320642.397864:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320642.779307:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320617/real 1767320617] req@ffff8ce1b3f9c700 x1853163985471104/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320642 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320642.779342:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3f9c700 x1853163985471104/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320642 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320642.779361:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320642.779381:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320642.779385:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985471104:0@lo:250:kworker.0 00000100:00080000:1.0:1767320642.779717:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320642.779723:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320642.779726:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320642.779731:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5865 00000100:00080000:1.0:1767320642.779734:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320642.779741:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320642.779774:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985477632/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320642.779799:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320642.779802:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320642.779804:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320642.779808:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5885 00000100:00080000:1.0:1767320642.779811:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320642.779814:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320642.779828:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6300 x1853163985477760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320642.779837:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320642.779839:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320642.779841:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320642.779844:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5885 00000100:00080000:1.0:1767320642.779846:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320642.779849:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320642.779861:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6d80 x1853163985477888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320642.779898:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320642.779901:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320642.779903:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320642.779907:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5885 00000100:00080000:1.0:1767320642.779909:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320642.779912:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320642.779924:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5f80 x1853163985478144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320642.779950:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320642.779953:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320642.779955:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320642.779958:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5885 00000100:00080000:1.0:1767320642.779961:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320642.779964:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320642.779973:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7800 x1853163985478400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320642.780037:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985477632:0@lo:250:kworker.0 00000400:00000100:2.0:1767320642.780081:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985477632 offset 0 length 520: 4 00000100:00100000:2.0:1767320642.780122:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985477760:0@lo:38:kworker.0 00000100:00100000:2.0:1767320642.780150:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.780185:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985477888:0@lo:38:kworker.0 00000100:00100000:2.0:1767320642.780245:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.780260:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985478144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320642.780279:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.780306:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985478400:0@lo:38:kworker.0 00000100:00100000:2.0:1767320642.780325:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.780357:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985477760 00000100:00100000:2.0:1767320642.780364:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985477760 00000100:00100000:2.0:1767320642.780393:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478144 00000100:00100000:2.0:1767320642.780396:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478144 00000100:00100000:2.0:1767320642.780407:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478400 00000100:00100000:2.0:1767320642.780409:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478400 00000100:00100000:2.0:1767320642.780423:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6062 00000100:00100000:2.0:1767320642.780429:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985477760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320642.780449:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320642.780499:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985477760:12345-0@lo:38:kworker.0 Request processed in 68us (348us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320642.780506:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6062 00000100:00100000:2.0:1767320642.780522:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6063 00000100:00100000:2.0:1767320642.780525:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985478144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320642.780534:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320642.780557:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985478144:12345-0@lo:38:kworker.0 Request processed in 30us (278us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320642.780563:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6063 00000100:00100000:2.0:1767320642.780574:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6064 00000100:00100000:2.0:1767320642.780577:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985478400:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320642.780586:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320642.780606:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985478400:12345-0@lo:38:kworker.0 Request processed in 27us (280us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320642.780612:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6064 00000100:00080000:2.0:1767320642.780641:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320642.780646:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320642.780649:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985477760:0@lo:38:kworker.0 00000100:00080000:2.0:1767320642.780656:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320642.780659:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320642.780661:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985478144:0@lo:38:kworker.0 00000100:00080000:2.0:1767320642.780668:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320642.780671:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320642.780673:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985478400:0@lo:38:kworker.0 00000100:00100000:2.0:1767320642.780710:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985478656:0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.780736:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.780754:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320642.780760:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985478016:0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.780779:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.780801:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478016 00000100:00100000:2.0:1767320642.780804:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478016 00000100:00100000:2.0:1767320642.780836:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5343 00000100:00100000:2.0:1767320642.780843:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985478016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.780880:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985478016:12345-0@lo:400:kworker.0 Request processed in 37us (99us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320642.780886:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5343 00000100:00100000:2.0:1767320642.780911:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985478016:0@lo:400:kworker.0 00000100:00080000:2.0:1767320642.780932:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320642.780950:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985478528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.780972:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.780989:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320642.780994:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985478272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.781013:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320642.781033:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478272 00000100:00100000:2.0:1767320642.781036:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478272 00000100:00100000:2.0:1767320642.781057:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5344 00000100:00100000:2.0:1767320642.781063:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985478272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.781092:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985478272:12345-0@lo:400:kworker.0 Request processed in 29us (78us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320642.781098:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5344 00000100:00080000:2.0:1767320642.781110:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320642.781124:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985478272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.781146:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478656 00000100:00100000:2.0:1767320642.781149:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478656 00000100:00100000:2.0:1767320642.781190:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478528 00000100:00100000:2.0:1767320642.781193:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478528 00000100:00100000:2.0:1767320642.781211:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3475 00000100:00100000:2.0:1767320642.781217:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985478656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.781249:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985478656:12345-0@lo:400:kworker.0 Request processed in 32us (512us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320642.781255:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3475 00000100:00100000:2.0:1767320642.781266:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3476 00000100:00100000:2.0:1767320642.781272:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985478528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.781300:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985478528:12345-0@lo:400:kworker.0 Request processed in 29us (327us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320642.781306:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3476 00000100:00100000:2.0:1767320642.781328:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985478528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320642.781348:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985478656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320642.781683:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985477888 00000100:00100000:0.0:1767320642.781688:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985477888 00000100:00100000:0.0:1767320642.781720:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2006 00000100:00100000:0.0:1767320642.781725:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985477888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320642.781739:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320642.781776:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985477888:12345-0@lo:38:kworker.0 Request processed in 49us (1530us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320642.781782:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2006 00000100:00080000:0.0:1767320642.781834:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320642.781838:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320642.781841:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985477888:0@lo:38:kworker.0 00010000:00010000:1.0:1767320643.422940:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320644.443504:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320645.469131:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320646.363521:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320646.363545:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985478784:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320646.363614:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320646.363881:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478784 00000100:00100000:1.0:1767320646.363890:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478784 00000100:00100000:1.0:1767320646.363949:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2676 00000100:00100000:1.0:1767320646.363959:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985478784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320646.363987:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320646.364030:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985478784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (416us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320646.364037:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2676 00000100:00100000:1.0:1767320646.364091:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985478784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320646.364119:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985478912:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320646.364144:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320646.364194:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985478912 00000100:00100000:1.0:1767320646.364198:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985478912 00000100:00100000:1.0:1767320646.364221:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2677 00000100:00100000:1.0:1767320646.364229:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985478912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320646.364244:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320646.364273:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985478912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (128us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320646.364280:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2677 00000100:00100000:1.0:1767320646.364308:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985478912:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320646.364440:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320646.364445:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320646.491855:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320646.827090:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320646.827138:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320646.827367:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920628736 00000100:00100000:1.0:1767320646.827378:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920628736 00000100:00100000:1.0:1767320646.827431:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920628864 00000100:00100000:1.0:1767320646.827435:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920628864 00000100:00100000:1.0:1767320646.827495:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5345 00000100:00100000:1.0:1767320646.827506:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920628864:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320646.827513:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6065 00000100:00100000:2.0:1767320646.827522:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920628736:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320646.827544:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320646.827680:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920628864:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 175us (539us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320646.827689:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5345 00000100:00080000:3.0:1767320646.827871:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320646.829364:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920628736:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1832us (2275us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320646.829375:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6065 00010000:00010000:1.0:1767320647.518487:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320647.899425:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320647.899435:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320647.899439:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320647.899443:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320647.899447:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320647.899454:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5891 00000100:00080000:0.0:1767320647.899459:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320647.899464:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320647.899502:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae300 x1853163985479040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320647.899526:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320647.899530:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320647.899531:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320647.899534:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5891 00000100:00080000:0.0:1767320647.899537:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320647.899540:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320647.899551:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99880 x1853163985479168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320647.899585:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320647.899588:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320647.899589:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320647.899592:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5891 00000100:00080000:0.0:1767320647.899595:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320647.899597:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320647.899606:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985479424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320647.899626:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320647.899629:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320647.899631:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320647.899634:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5891 00000100:00080000:0.0:1767320647.899636:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320647.899638:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320647.899645:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3800 x1853163985479680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320647.899760:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985479936:0@lo:400:kworker.0 00000100:00100000:3.0:1767320647.899818:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320647.899885:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320647.899891:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985479296:0@lo:400:kworker.0 00000100:00100000:3.0:1767320647.899913:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320647.899952:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479296 00000100:00100000:3.0:1767320647.899959:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479296 00000100:00100000:3.0:1767320647.900009:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5346 00000100:00100000:3.0:1767320647.900019:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985479296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320647.900067:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985479296:12345-0@lo:400:kworker.0 Request processed in 49us (152us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320647.900073:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5346 00000100:00080000:3.0:1767320647.900095:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320647.900123:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985479296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320647.900233:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479936 00000100:00100000:0.0:1767320647.900237:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479936 00000100:00100000:0.0:1767320647.900273:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3477 00000100:00100000:0.0:1767320647.900279:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985479936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320647.900314:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985479936:12345-0@lo:400:kworker.0 Request processed in 35us (495us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320647.900320:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3477 00000100:00100000:2.0:1767320647.900482:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985479808:0@lo:400:kworker.0 00000100:00100000:2.0:1767320647.900511:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320647.900537:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320647.900543:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985479552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320647.900565:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320647.900594:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479552 00000100:00100000:2.0:1767320647.900598:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479552 00000100:00100000:2.0:1767320647.900627:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5347 00000100:00100000:2.0:1767320647.900633:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985479552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320647.900663:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985479552:12345-0@lo:400:kworker.0 Request processed in 31us (97us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320647.900669:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5347 00000100:00080000:2.0:1767320647.900687:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320647.900704:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985479552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320647.900722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985479040:0@lo:38:kworker.0 00000100:00100000:2.0:1767320647.900758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320647.900779:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985479168:0@lo:38:kworker.0 00000100:00100000:2.0:1767320647.900803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320647.900823:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985479424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320647.900843:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320647.900868:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985479680:0@lo:38:kworker.0 00000100:00100000:2.0:1767320647.900886:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320647.900917:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479168 00000100:00100000:2.0:1767320647.900921:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479168 00000100:00100000:2.0:1767320647.900941:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2007 00000100:00100000:2.0:1767320647.900945:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985479168:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320647.900964:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320647.900995:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985479168:12345-0@lo:38:kworker.0 Request processed in 48us (190us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320647.901002:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2007 00000100:00100000:2.0:1767320647.901018:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479040 00000100:00100000:2.0:1767320647.901021:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479040 00000100:00100000:2.0:1767320647.901036:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479424 00000100:00100000:2.0:1767320647.901038:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479424 00000100:00100000:2.0:1767320647.901050:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479680 00000100:00100000:2.0:1767320647.901053:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479680 00000100:00100000:2.0:1767320647.901063:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6066 00000100:00100000:2.0:1767320647.901068:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985479040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320647.901077:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320647.901267:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6067 00000100:00100000:0.0:1767320647.901271:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985479424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320647.901280:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320647.901347:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6068 00000100:00100000:1.0:1767320647.901352:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985479680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320647.901362:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320647.901915:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985479808 00000100:00100000:3.0:1767320647.901920:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985479808 00000100:00100000:3.0:1767320647.901956:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3478 00000100:00100000:3.0:1767320647.901962:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985479808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320647.902588:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985479424:12345-0@lo:38:kworker.0 Request processed in 1312us (1741us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320647.902596:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6067 00000100:00100000:0.0:1767320647.902726:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985479936:0@lo:400:kworker.0 00000100:00080000:0.0:1767320647.902780:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320647.902786:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320647.902789:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985479040:0@lo:38:kworker.0 00000100:00080000:0.0:1767320647.902798:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320647.902802:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320647.902805:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985479168:0@lo:38:kworker.0 00000100:00080000:0.0:1767320647.902813:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320647.902816:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320647.902819:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985479424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320647.903354:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985479680:12345-0@lo:38:kworker.0 Request processed in 1997us (2464us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320647.903362:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6068 00000100:00080000:0.0:1767320647.903440:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320647.903444:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320647.903447:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985479680:0@lo:38:kworker.0 00000100:00100000:2.0:1767320647.904086:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985479040:12345-0@lo:38:kworker.0 Request processed in 3012us (3323us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320647.904096:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6066 00000100:00100000:3.0:1767320647.904650:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985479808:12345-0@lo:400:kworker.0 Request processed in 2685us (4136us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320647.904659:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3478 00000100:00100000:2.0:1767320647.904856:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985479808:0@lo:400:kworker.0 00010000:00010000:1.0:1767320648.547743:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320649.563411:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320650.592439:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320651.487193:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320651.487222:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985480064:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320651.487290:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320651.487436:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480064 00000100:00100000:3.0:1767320651.487444:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480064 00000100:00100000:3.0:1767320651.487502:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2678 00000100:00100000:3.0:1767320651.487512:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985480064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320651.487543:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320651.487592:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985480064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (301us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320651.487600:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2678 00000100:00100000:3.0:1767320651.487648:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985480064:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320651.487712:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320651.487889:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320651.487898:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985480192:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320651.487926:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320651.488031:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480192 00000100:00100000:3.0:1767320651.488035:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480192 00000100:00100000:3.0:1767320651.488074:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2679 00000100:00100000:3.0:1767320651.488081:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985480192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320651.488099:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320651.488137:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985480192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (210us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320651.488143:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2679 00000100:00100000:1.0:1767320651.488208:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985480192:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320651.488255:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320651.611410:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320651.949585:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320651.949652:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320651.949755:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920629120 00000100:00100000:1.0:1767320651.949766:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920629120 00000100:00100000:1.0:1767320651.949860:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920628992 00000100:00100000:1.0:1767320651.949864:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920628992 00000100:00100000:1.0:1767320651.949886:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5348 00000100:00100000:1.0:1767320651.949897:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920629120:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320651.949974:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920629120:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (399us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320651.949982:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5348 00000100:00100000:1.0:1767320651.950001:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6069 00000100:00100000:1.0:1767320651.950005:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920628992:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320651.950026:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320651.950049:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920628992:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (397us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320651.950056:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6069 00000100:00080000:3.0:1767320651.950382:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320652.639024:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320653.019370:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320653.019383:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320653.019387:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320653.019392:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320653.019398:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320653.019409:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5896 00000100:00080000:0.0:1767320653.019415:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320653.019424:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320653.019474:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2300 x1853163985480320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320653.019494:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320653.019498:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320653.019500:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320653.019503:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5896 00000100:00080000:0.0:1767320653.019506:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320653.019511:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320653.019523:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985480448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320653.019576:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320653.019579:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320653.019581:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320653.019584:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5896 00000100:00080000:0.0:1767320653.019587:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320653.019590:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320653.019603:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0700 x1853163985480704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320653.019627:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320653.019631:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320653.019632:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320653.019636:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5896 00000100:00080000:0.0:1767320653.019638:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320653.019641:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320653.019651:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae300 x1853163985480960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320653.019863:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985481088:0@lo:400:kworker.0 00000100:00100000:1.0:1767320653.019960:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320653.020040:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320653.020047:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985480832:0@lo:400:kworker.0 00000100:00100000:1.0:1767320653.020073:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320653.020118:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985480320:0@lo:38:kworker.0 00000100:00100000:1.0:1767320653.020123:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480832 00000100:00100000:1.0:1767320653.020133:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480832 00000100:00100000:2.0:1767320653.020190:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320653.020223:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985480448:0@lo:38:kworker.0 00000100:00100000:1.0:1767320653.020238:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480320 00000100:00100000:1.0:1767320653.020243:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480320 00000100:00100000:2.0:1767320653.020250:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320653.020270:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5349 00000100:00100000:2.0:1767320653.020283:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985480704:0@lo:38:kworker.0 00000100:00100000:1.0:1767320653.020284:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985480832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320653.020320:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320653.020340:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985480960:0@lo:38:kworker.0 00000100:00100000:1.0:1767320653.020367:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985480832:12345-0@lo:400:kworker.0 Request processed in 88us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320653.020375:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320653.020381:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5349 00000100:00100000:1.0:1767320653.020401:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480704 00000100:00100000:1.0:1767320653.020406:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480704 00000100:00100000:1.0:1767320653.020421:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480960 00000100:00100000:1.0:1767320653.020423:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480960 00000100:00100000:2.0:1767320653.020427:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6070 00000100:00100000:2.0:1767320653.020434:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985480320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320653.020452:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320653.020479:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320653.020489:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985480320:12345-0@lo:38:kworker.0 Request processed in 54us (324us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320653.020498:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985481216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320653.020541:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481088 00000100:00100000:0.0:1767320653.020545:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481088 00000100:00100000:0.0:1767320653.020578:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3479 00000100:00100000:0.0:1767320653.020583:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985481088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320653.020619:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985481088:12345-0@lo:400:kworker.0 Request processed in 36us (660us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320653.020628:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3479 00000100:00100000:0.0:1767320653.020667:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985480576:0@lo:400:kworker.0 00000100:00100000:0.0:1767320653.020692:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320653.020714:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480576 00000100:00100000:0.0:1767320653.020717:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480576 00000100:00100000:3.0:1767320653.020777:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320653.020806:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481216 00000100:00100000:3.0:1767320653.020810:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481216 00000100:00100000:3.0:1767320653.020834:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3480 00000100:00100000:3.0:1767320653.020840:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985481216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320653.020876:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985481216:12345-0@lo:400:kworker.0 Request processed in 35us (98us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320653.020882:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3480 00000100:00080000:3.0:1767320653.020897:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320653.020899:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320653.020901:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320653.020938:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985481216:0@lo:400:kworker.0 00000100:00100000:1.0:1767320653.021521:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6071 00000100:00100000:1.0:1767320653.021528:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985480704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320653.021545:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320653.021588:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985480704:12345-0@lo:38:kworker.0 Request processed in 58us (1268us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320653.021594:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6071 00000100:00100000:1.0:1767320653.021607:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6072 00000100:00100000:1.0:1767320653.021611:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985480960:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320653.021620:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320653.021643:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985480960:12345-0@lo:38:kworker.0 Request processed in 30us (1272us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320653.021649:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6072 00000100:00100000:1.0:1767320653.021684:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985480832:0@lo:400:kworker.0 00000100:00100000:1.0:1767320653.021697:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985481088:0@lo:400:kworker.0 00000100:00080000:1.0:1767320653.021747:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320653.021753:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320653.021757:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985480320:0@lo:38:kworker.0 00000100:00080000:1.0:1767320653.021766:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320653.021770:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320653.021773:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985480704:0@lo:38:kworker.0 00000100:00080000:1.0:1767320653.021780:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320653.021784:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320653.021787:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985480960:0@lo:38:kworker.0 00000100:00100000:1.0:1767320653.021826:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985480448 00000100:00100000:1.0:1767320653.021831:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985480448 00000100:00100000:1.0:1767320653.021854:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2008 00000100:00100000:1.0:1767320653.021858:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985480448:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320653.021869:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320653.021901:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985480448:12345-0@lo:38:kworker.0 Request processed in 41us (1651us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320653.021921:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2008 00000100:00080000:1.0:1767320653.021944:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320653.021948:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320653.021951:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985480448:0@lo:38:kworker.0 00000100:00100000:2.0:1767320653.028196:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6070 00000100:00100000:0.0:1767320653.034246:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5350 00000100:00100000:0.0:1767320653.034271:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985480576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320653.034395:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985480576:12345-0@lo:400:kworker.0 Request processed in 136us (13700us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320653.034404:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5350 00000100:00100000:2.0:1767320653.034467:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985480576:0@lo:400:kworker.0 00010000:00010000:1.0:1767320653.659446:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320654.683572:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320655.707732:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320656.603549:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985481344:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320656.603629:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320656.603803:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481344 00000100:00100000:0.0:1767320656.603812:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481344 00000100:00100000:0.0:1767320656.603900:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2680 00000100:00100000:0.0:1767320656.603913:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985481344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320656.603948:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320656.603994:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985481344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (365us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320656.604001:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2680 00000100:00080000:1.0:1767320656.604059:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320656.604139:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985481344:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320656.607517:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985481472:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320656.607597:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320656.607755:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481472 00000100:00100000:3.0:1767320656.607765:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481472 00000100:00100000:3.0:1767320656.607834:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2681 00000100:00100000:3.0:1767320656.607843:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985481472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320656.607882:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320656.607954:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985481472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (356us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320656.607962:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2681 00000100:00100000:2.0:1767320656.608059:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985481472:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320656.608189:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320656.732557:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320657.069655:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320657.069944:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920629376 00000100:00100000:3.0:1767320657.069965:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920629376 00000100:00100000:3.0:1767320657.070134:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5351 00000100:00100000:3.0:1767320657.070147:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920629376:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320657.070332:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920629376:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 188us (688us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320657.070341:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5351 00000100:00100000:0.0:1767320657.073418:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320657.073509:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920629248 00000100:00100000:3.0:1767320657.073518:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920629248 00000100:00100000:3.0:1767320657.073606:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6073 00000100:00100000:3.0:1767320657.073613:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920629248:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320657.073639:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320657.073712:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920629248:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 97us (302us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320657.073719:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6073 00000100:00080000:1.0:1767320657.074493:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320657.758697:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320658.139595:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320658.139607:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320658.139610:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320658.139614:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320658.139619:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320658.139627:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5901 00000100:00080000:0.0:1767320658.139632:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320658.139640:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320658.139682:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4000 x1853163985481600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320658.139713:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320658.139717:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320658.139718:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320658.139722:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5901 00000100:00080000:0.0:1767320658.139724:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320658.139728:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320658.139741:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5f80 x1853163985481728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320658.139787:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320658.139791:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320658.139793:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320658.139796:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5901 00000100:00080000:0.0:1767320658.139799:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320658.139802:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320658.139813:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bea300 x1853163985481984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320658.139833:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320658.139835:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320658.139837:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320658.139840:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5901 00000100:00080000:0.0:1767320658.139843:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320658.139846:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320658.139854:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1180 x1853163985482240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320658.140028:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985482368:0@lo:400:kworker.0 00000100:00100000:2.0:1767320658.140112:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320658.140207:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320658.140214:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985482112:0@lo:400:kworker.0 00000100:00100000:2.0:1767320658.140239:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320658.140272:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985481856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320658.140292:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320658.140306:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320658.140310:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985482496:0@lo:400:kworker.0 00000100:00100000:2.0:1767320658.140326:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320658.140488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985481600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320658.140518:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320658.140548:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985481728:0@lo:38:kworker.0 00000100:00100000:1.0:1767320658.140572:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320658.140596:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985481984:0@lo:38:kworker.0 00000100:00100000:1.0:1767320658.140617:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320658.140634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985482240:0@lo:38:kworker.0 00000100:00100000:1.0:1767320658.140652:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320658.140683:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985482112 00000100:00100000:1.0:1767320658.140691:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985482112 00000100:00100000:1.0:1767320658.140739:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481856 00000100:00100000:1.0:1767320658.140743:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481856 00000100:00100000:1.0:1767320658.140764:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481600 00000100:00100000:2.0:1767320658.140764:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481984 00000100:00100000:1.0:1767320658.140767:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481600 00000100:00100000:2.0:1767320658.140772:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481984 00000100:00100000:1.0:1767320658.140781:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985482240 00000100:00100000:1.0:1767320658.140785:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985482240 00000100:00100000:2.0:1767320658.140799:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5352 00000100:00100000:0.0:1767320658.140849:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985482368 00000100:00100000:0.0:1767320658.140853:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985482368 00000100:00100000:0.0:1767320658.140882:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985482496 00000100:00100000:0.0:1767320658.140885:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985482496 00000100:00100000:0.0:1767320658.140907:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3481 00000100:00100000:0.0:1767320658.140915:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985482368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320658.140971:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985482368:12345-0@lo:400:kworker.0 Request processed in 56us (858us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320658.140978:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3481 00000100:00100000:0.0:1767320658.140995:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3482 00000100:00100000:0.0:1767320658.141001:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985482496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320658.141310:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985482496:12345-0@lo:400:kworker.0 Request processed in 31us (704us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320658.141319:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3482 00000100:00100000:2.0:1767320658.141366:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985482112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320658.141397:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985482112:12345-0@lo:400:kworker.0 Request processed in 590us (1157us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320658.147324:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5353 00000100:00100000:1.0:1767320658.147355:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985481856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320658.147461:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985481856:12345-0@lo:400:kworker.0 Request processed in 124us (7167us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320658.147470:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5353 00000100:00100000:1.0:1767320658.147508:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6074 00000100:00100000:1.0:1767320658.147512:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985481600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320658.147532:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320658.147577:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985481600:12345-0@lo:38:kworker.0 Request processed in 63us (7058us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320658.147583:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6074 00000100:00100000:1.0:1767320658.147599:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6075 00000100:00100000:1.0:1767320658.147603:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985481984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320658.147613:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320658.147637:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985481984:12345-0@lo:38:kworker.0 Request processed in 33us (7020us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320658.147644:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6075 00000100:00100000:1.0:1767320658.147658:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6076 00000100:00100000:1.0:1767320658.147662:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985482240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320658.147671:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320658.147693:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985482240:12345-0@lo:38:kworker.0 Request processed in 29us (7039us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320658.147698:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6076 00000100:00100000:1.0:1767320658.147728:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985481728 00000100:00100000:1.0:1767320658.147736:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985481728 00000100:00100000:1.0:1767320658.147764:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2009 00000100:00100000:1.0:1767320658.147769:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985481728:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320658.147779:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320658.147802:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985481728:12345-0@lo:38:kworker.0 Request processed in 32us (7229us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320658.147807:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2009 00000100:00080000:1.0:1767320658.147831:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320658.147836:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320658.147838:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320658.147878:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320658.147884:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320658.147888:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985481600:0@lo:38:kworker.0 00000100:00080000:1.0:1767320658.147899:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320658.147903:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320658.147906:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985481728:0@lo:38:kworker.0 00000100:00080000:1.0:1767320658.147915:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320658.147919:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320658.147923:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985481984:0@lo:38:kworker.0 00000100:00080000:1.0:1767320658.147931:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320658.147934:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320658.147936:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985482240:0@lo:38:kworker.0 00000100:00100000:1.0:1767320658.148024:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985482112:0@lo:400:kworker.0 00000100:00100000:1.0:1767320658.148035:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985482368:0@lo:400:kworker.0 00000100:00100000:1.0:1767320658.148099:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985482496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320658.148109:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985481856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320658.157858:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5352 00010000:00010000:1.0:1767320658.781517:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320659.803781:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320660.827479:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320661.724996:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985482624:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320661.725090:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320661.727913:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985482624 00000100:00100000:0.0:1767320661.727924:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985482624 00000100:00100000:0.0:1767320661.727993:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2682 00000100:00100000:0.0:1767320661.728000:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985482624:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320661.728035:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320661.728148:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320661.728187:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985482752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320661.728240:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320661.728289:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985482752 00000100:00100000:2.0:1767320661.728293:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985482752 00000100:00100000:2.0:1767320661.728324:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2683 00000100:00100000:2.0:1767320661.728332:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985482752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320661.728350:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320661.728381:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985482752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (141us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320661.728387:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2683 00000100:00100000:2.0:1767320661.728432:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985482752:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320661.728478:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320661.728482:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320661.730714:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985482624:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2702us (5611us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320661.730731:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2682 00000100:00100000:1.0:1767320661.734684:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985482624:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320661.851451:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320662.186076:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320662.186407:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320662.189101:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920629632 00000100:00100000:2.0:1767320662.189143:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920629632 00000100:00100000:2.0:1767320662.189287:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920629504 00000100:00100000:2.0:1767320662.189292:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920629504 00000100:00100000:2.0:1767320662.189325:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5354 00000100:00100000:2.0:1767320662.189337:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920629632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767320662.190045:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320662.190149:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920629632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 793us (4067us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320662.190189:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5354 00000100:00100000:2.0:1767320662.190227:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6077 00000100:00100000:2.0:1767320662.190233:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920629504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320662.190258:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320662.192180:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920629504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1914us (5742us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320662.192198:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6077 00010000:00010000:2.0:1767320662.876821:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320663.262961:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320663.262973:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320663.262977:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320663.262982:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320663.262988:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320663.262999:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5906 00000100:00080000:1.0:1767320663.263005:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320663.263015:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320663.263065:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985482880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320663.263132:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320663.263137:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320663.263139:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320663.263144:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5906 00000100:00080000:1.0:1767320663.263146:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320663.263151:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320663.263190:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5180 x1853163985483008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320663.263247:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320663.263252:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320663.263254:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320663.263258:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5906 00000100:00080000:1.0:1767320663.263261:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320663.263264:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320663.263278:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7480 x1853163985483264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320663.263305:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320663.263309:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320663.263311:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320663.263314:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5906 00000100:00080000:1.0:1767320663.263317:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320663.263321:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320663.263334:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7f480 x1853163985483520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320663.263378:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985483776:0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.263470:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320663.263549:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320663.263557:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985483136:0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.263588:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320663.263629:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483776 00000100:00100000:1.0:1767320663.263639:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483776 00000100:00100000:1.0:1767320663.263707:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3483 00000100:00100000:1.0:1767320663.263718:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985483776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.263781:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985483776:12345-0@lo:400:kworker.0 Request processed in 65us (311us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320663.263789:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3483 00000100:00080000:1.0:1767320663.263817:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320663.263852:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985483776:0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.263879:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985483392:0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.263902:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320663.263923:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320663.263928:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985483648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.263949:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320663.263970:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483648 00000100:00100000:1.0:1767320663.263973:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483648 00000100:00100000:1.0:1767320663.263997:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3484 00000100:00100000:1.0:1767320663.264003:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985483648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.264035:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985483648:12345-0@lo:400:kworker.0 Request processed in 32us (85us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320663.264040:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3484 00000100:00080000:1.0:1767320663.264052:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320663.264067:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985483648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.264088:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483136 00000100:00100000:1.0:1767320663.264091:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483136 00000100:00100000:1.0:1767320663.264139:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483392 00000100:00100000:1.0:1767320663.264142:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483392 00000100:00100000:1.0:1767320663.264160:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5355 00000100:00100000:1.0:1767320663.264195:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985483136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.264232:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985483136:12345-0@lo:400:kworker.0 Request processed in 38us (643us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320663.264238:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5355 00000100:00100000:1.0:1767320663.264250:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5356 00000100:00100000:1.0:1767320663.264256:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985483392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.264286:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985483392:12345-0@lo:400:kworker.0 Request processed in 30us (383us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320663.264292:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5356 00000100:00080000:1.0:1767320663.264305:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320663.264322:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985483136:0@lo:400:kworker.0 00000100:00100000:1.0:1767320663.264347:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985483392:0@lo:400:kworker.0 00000100:00100000:2.0:1767320663.265974:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985482880:0@lo:38:kworker.0 00000100:00100000:2.0:1767320663.266048:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320663.266094:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985483008:0@lo:38:kworker.0 00000100:00100000:2.0:1767320663.266136:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320663.266193:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985483264:0@lo:38:kworker.0 00000100:00100000:2.0:1767320663.266224:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320663.266240:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985483520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320663.266259:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320663.266301:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483008 00000100:00100000:2.0:1767320663.266313:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483008 00000100:00100000:2.0:1767320663.266357:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2010 00000100:00100000:2.0:1767320663.266365:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985483008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320663.266392:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320663.266434:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985483008:12345-0@lo:38:kworker.0 Request processed in 68us (297us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320663.266445:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2010 00000100:00100000:2.0:1767320663.266470:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985482880 00000100:00100000:2.0:1767320663.266473:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985482880 00000100:00100000:2.0:1767320663.266490:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483264 00000100:00100000:2.0:1767320663.266492:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483264 00000100:00100000:2.0:1767320663.266503:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483520 00000100:00100000:2.0:1767320663.266505:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483520 00000100:00100000:2.0:1767320663.266517:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6078 00000100:00100000:2.0:1767320663.266521:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985482880:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320663.266531:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320663.266552:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985482880:12345-0@lo:38:kworker.0 Request processed in 30us (504us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320663.266558:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6078 00000100:00100000:2.0:1767320663.266567:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6079 00000100:00100000:2.0:1767320663.266570:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985483264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320663.266578:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320663.266593:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985483264:12345-0@lo:38:kworker.0 Request processed in 22us (369us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320663.266599:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6079 00000100:00100000:2.0:1767320663.266606:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6080 00000100:00100000:2.0:1767320663.266610:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985483520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320663.266618:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320663.266632:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985483520:12345-0@lo:38:kworker.0 Request processed in 21us (373us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320663.266637:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6080 00000100:00080000:2.0:1767320663.266678:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320663.266684:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320663.266688:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985482880:0@lo:38:kworker.0 00000100:00080000:2.0:1767320663.266697:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320663.266700:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320663.266703:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985483008:0@lo:38:kworker.0 00000100:00080000:2.0:1767320663.266711:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320663.266714:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320663.266717:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985483264:0@lo:38:kworker.0 00000100:00080000:2.0:1767320663.266724:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320663.266727:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320663.266730:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985483520:0@lo:38:kworker.0 00010000:00010000:2.0:1767320663.899682:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320664.924513:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320665.947997:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320666.287707:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767320666.287749:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920629760 offset 0 length 520: 4 00000100:00100000:0.0:1767320666.287778:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320666.287916:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630016 00000100:00100000:1.0:1767320666.287928:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630016 00000100:00100000:1.0:1767320666.288023:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920629888 00000100:00100000:1.0:1767320666.288027:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920629888 00000100:00100000:1.0:1767320666.288045:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5357 00000100:00100000:1.0:1767320666.288082:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920630016:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320666.288188:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920630016:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (469us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320666.288198:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5357 00000100:00100000:1.0:1767320666.288218:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6081 00000100:00100000:1.0:1767320666.288222:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920629888:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320666.288242:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320666.288268:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920629888:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (489us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320666.288274:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6081 00000100:00080000:3.0:1767320666.289269:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320666.843751:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985483904:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320666.843815:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320666.844030:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320666.844071:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985484032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320666.844127:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985483904 00000100:00100000:2.0:1767320666.844138:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985483904 00000100:00100000:3.0:1767320666.844148:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320666.844229:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985484032 00000100:00100000:2.0:1767320666.844234:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985484032 00000100:00100000:2.0:1767320666.844271:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2684 00000100:00100000:2.0:1767320666.844285:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985483904:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320666.844315:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320666.844329:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2685 00000100:00100000:3.0:1767320666.844348:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985484032:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320666.844364:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985483904:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (545us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320666.844370:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2684 00002000:00000024:3.0:1767320666.844384:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767320666.844440:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320666.844447:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320666.844457:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985484032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 116us (308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320666.844467:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2685 00000100:00100000:1.0:1767320666.844486:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985483904:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320666.844563:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985484032:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320666.971505:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320667.355254:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320642/real 1767320642] req@ffff8ce1a54b4a80 x1853163985477632/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320667 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320667.355294:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a54b4a80 x1853163985477632/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320667 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320667.355316:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320667.355339:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320667.355423:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985477632:0@lo:250:kworker.0 00000100:00080000:1.0:1767320667.356964:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320667.356976:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320667.356981:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320667.356992:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5891 00000100:00080000:1.0:1767320667.356997:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320667.357007:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320667.357069:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4000 x1853163985484160/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320667.357107:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320667.357111:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320667.357113:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320667.357117:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5911 00000100:00080000:1.0:1767320667.357120:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320667.357123:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320667.357145:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985484288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320667.357156:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320667.357160:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320667.357187:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320667.357193:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5911 00000100:00080000:1.0:1767320667.357196:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320667.357201:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320667.357228:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985484160:0@lo:250:kworker.0 00000100:00100000:1.0:1767320667.357228:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b7480 x1853163985484416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320667.357292:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000100:0.0:1767320667.357298:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985484160 offset 0 length 520: 4 00000100:00080000:1.0:1767320667.357302:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320667.357308:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320667.357316:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5911 00000100:00080000:1.0:1767320667.357320:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320667.357325:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320667.357346:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4700 x1853163985484672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320667.357363:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985484288:0@lo:38:kworker.0 00000100:00080000:1.0:1767320667.357384:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320667.357389:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320667.357392:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320667.357402:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5911 00000100:00100000:0.0:1767320667.357403:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320667.357411:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320667.357417:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320667.357436:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985484416:0@lo:38:kworker.0 00000100:00100000:1.0:1767320667.357436:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4000 x1853163985484928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320667.357476:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320667.357493:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320667.357500:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985484672:0@lo:38:kworker.0 00000100:00100000:1.0:1767320667.357516:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985484800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320667.357537:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320667.357558:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985484928:0@lo:38:kworker.0 00000100:00100000:1.0:1767320667.357586:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320667.357600:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985484416 00000100:00100000:0.0:1767320667.357608:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985484416 00000100:00100000:1.0:1767320667.357649:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320667.357654:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2011 00000100:00100000:0.0:1767320667.357663:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985484416:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320667.357668:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985484544:0@lo:400:kworker.0 00010000:00020000:0.0:1767320667.357686:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320667.357712:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320667.357727:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985484416:12345-0@lo:38:kworker.0 Request processed in 64us (250us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320667.357732:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2011 00000100:00100000:0.0:1767320667.360378:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985484288 00000100:00100000:0.0:1767320667.360393:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985484288 00000100:00100000:0.0:1767320667.360443:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985484672 00000100:00100000:0.0:1767320667.360446:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985484672 00000100:00100000:0.0:1767320667.360461:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985484800 00000100:00100000:0.0:1767320667.360463:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985484800 00000100:00100000:0.0:1767320667.360498:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985484544 00000100:00100000:0.0:1767320667.360501:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985484544 00000100:00100000:0.0:1767320667.360524:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5358 00000100:00100000:0.0:1767320667.360533:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985484800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320667.360624:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985484800:12345-0@lo:400:kworker.0 Request processed in 91us (3040us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320667.360631:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5358 00000100:00100000:0.0:1767320667.360647:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5359 00000100:00100000:0.0:1767320667.360653:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985484544:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320667.360687:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985484544:12345-0@lo:400:kworker.0 Request processed in 34us (2978us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320667.360693:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5359 00000100:00100000:0.0:1767320667.360705:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6082 00000100:00100000:0.0:1767320667.360709:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985484288:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320667.360729:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320667.360756:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985484288:12345-0@lo:38:kworker.0 Request processed in 46us (3360us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320667.360762:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6082 00000100:00100000:0.0:1767320667.360774:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6083 00000100:00100000:0.0:1767320667.360778:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985484672:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320667.360787:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320667.360809:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985484672:12345-0@lo:38:kworker.0 Request processed in 30us (3273us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320667.360815:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6083 00000100:00080000:0.0:1767320667.360845:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320667.360897:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985484800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320667.360934:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985484544:0@lo:400:kworker.0 00000100:00100000:0.0:1767320667.360961:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320667.360997:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320667.361002:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320667.361007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985484416:0@lo:38:kworker.0 00000100:00100000:1.0:1767320667.361090:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985484928 00000100:00100000:1.0:1767320667.361103:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985484928 00000100:00100000:1.0:1767320667.361152:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6084 00000100:00100000:1.0:1767320667.361159:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985484928:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320667.361192:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320667.361221:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320667.361229:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320667.361234:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985484672:0@lo:38:kworker.0 00000100:00080000:0.0:1767320667.361252:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320667.361259:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320667.361264:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985484288:0@lo:38:kworker.0 00000100:00100000:1.0:1767320667.361268:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985484928:12345-0@lo:38:kworker.0 Request processed in 106us (304us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320667.361278:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6084 00000100:00080000:0.0:1767320667.361309:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320667.361315:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320667.361319:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985484928:0@lo:38:kworker.0 00010000:00010000:1.0:1767320667.995954:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320669.022204:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320670.043607:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320671.067426:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320671.403843:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320671.403944:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320671.404027:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630272 00000100:00100000:2.0:1767320671.404040:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630272 00000100:00100000:0.0:1767320671.404045:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630144 00000100:00100000:0.0:1767320671.404048:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630144 00000100:00100000:0.0:1767320671.404092:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5360 00000100:00100000:2.0:1767320671.404098:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6085 00000100:00100000:0.0:1767320671.404102:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920630272:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320671.404103:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920630144:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320671.404125:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320671.404198:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320671.404331:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920630144:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 224us (385us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320671.404338:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6085 00000100:00100000:0.0:1767320671.410042:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920630272:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 5926us (6195us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320671.410063:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5360 00000100:00100000:0.0:1767320671.963761:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985485056:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320671.963834:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320671.963975:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485056 00000100:00100000:3.0:1767320671.963984:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485056 00000100:00100000:3.0:1767320671.964048:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2686 00000100:00100000:3.0:1767320671.964058:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985485056:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320671.964086:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320671.964131:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985485056:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (295us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320671.964139:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2686 00000100:00100000:3.0:1767320671.964209:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985485056:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320671.964565:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320671.966737:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985485184:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320671.966806:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320671.966922:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485184 00000100:00100000:0.0:1767320671.966930:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485184 00000100:00100000:0.0:1767320671.966998:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2687 00000100:00100000:0.0:1767320671.967007:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985485184:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320671.967035:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320671.967077:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985485184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (270us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320671.967084:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2687 00000100:00080000:1.0:1767320671.967539:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320671.967576:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985485184:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320672.094586:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320672.483073:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320672.483084:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320672.483097:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320672.483102:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320672.483106:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320672.483113:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5915 00000100:00080000:3.0:1767320672.483118:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320672.483125:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320672.483190:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7c000 x1853163985485312/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320672.483214:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320672.483218:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320672.483220:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767320672.483533:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985485312:0@lo:38:kworker.0 00000100:00100000:0.0:1767320672.483615:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320672.483628:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5915 00000100:00080000:3.0:1767320672.483631:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320672.483635:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320672.483661:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7d880 x1853163985485440/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320672.483687:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985485440:0@lo:38:kworker.0 00000100:00100000:0.0:1767320672.483713:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320672.483771:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485440 00000100:00100000:0.0:1767320672.483777:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485440 00000100:00080000:3.0:1767320672.483777:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320672.483780:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320672.483783:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320672.483787:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5915 00000100:00080000:3.0:1767320672.483789:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320672.483793:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320672.483804:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7ed80 x1853163985485696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320672.483811:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2012 00000100:00100000:0.0:1767320672.483819:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985485440:12345-0@lo:38:kworker.0 00000100:00080000:3.0:1767320672.483827:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320672.483830:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320672.483832:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320672.483835:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5915 00000100:00080000:3.0:1767320672.483837:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320672.483840:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00010000:00020000:0.0:1767320672.483842:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320672.483850:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99880 x1853163985485952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320672.483873:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985485568:0@lo:400:kworker.0 00000100:00100000:0.0:1767320672.483875:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985485440:12345-0@lo:38:kworker.0 Request processed in 56us (160us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320672.483881:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2012 00000100:00100000:3.0:1767320672.483898:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320672.483905:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985485696:0@lo:38:kworker.0 00000100:00100000:0.0:1767320672.483925:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320672.483932:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320672.483952:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985485952:0@lo:38:kworker.0 00000100:00100000:3.0:1767320672.483967:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985485824:0@lo:400:kworker.0 00000100:00100000:0.0:1767320672.483972:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320672.483990:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320672.484008:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320672.484013:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320672.484017:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985485440:0@lo:38:kworker.0 00000100:00100000:3.0:1767320672.484022:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485312 00000100:00100000:3.0:1767320672.484025:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485312 00000100:00100000:3.0:1767320672.484049:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485568 00000100:00100000:3.0:1767320672.484052:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485568 00000100:00100000:0.0:1767320672.484053:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485696 00000100:00100000:0.0:1767320672.484055:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485696 00000100:00100000:0.0:1767320672.484066:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485952 00000100:00100000:0.0:1767320672.484069:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485952 00000100:00100000:0.0:1767320672.484074:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985485824 00000100:00100000:0.0:1767320672.484076:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985485824 00000100:00100000:3.0:1767320672.484082:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5361 00000100:00100000:3.0:1767320672.484953:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985485568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320672.485025:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985485568:12345-0@lo:400:kworker.0 Request processed in 93us (1125us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320672.490572:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5362 00000100:00100000:0.0:1767320672.490598:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985485824:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320672.490705:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985485824:12345-0@lo:400:kworker.0 Request processed in 121us (6713us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320672.490714:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5362 00000100:00100000:0.0:1767320672.490735:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6086 00000100:00100000:0.0:1767320672.490739:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985485312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320672.490760:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320672.490791:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985485312:12345-0@lo:38:kworker.0 Request processed in 50us (7177us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320672.490798:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6086 00000100:00100000:0.0:1767320672.490812:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6087 00000100:00100000:0.0:1767320672.490816:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985485696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320672.490825:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320672.490846:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985485696:12345-0@lo:38:kworker.0 Request processed in 29us (6920us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320672.490852:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6087 00000100:00100000:0.0:1767320672.490861:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6088 00000100:00100000:0.0:1767320672.490866:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985485952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320672.490875:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320672.490895:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985485952:12345-0@lo:38:kworker.0 Request processed in 28us (6921us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320672.490901:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6088 00000100:00080000:0.0:1767320672.490954:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320672.490960:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320672.490964:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985485696:0@lo:38:kworker.0 00000100:00080000:0.0:1767320672.490975:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320672.490978:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320672.490981:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985485952:0@lo:38:kworker.0 00000100:00080000:0.0:1767320672.490989:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320672.490992:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320672.490994:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985485312:0@lo:38:kworker.0 00000100:00080000:0.0:1767320672.491094:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320672.491129:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985485568:0@lo:400:kworker.0 00000100:00100000:0.0:1767320672.491159:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985485824:0@lo:400:kworker.0 00000100:00100000:3.0:1767320672.494987:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5361 00010000:00010000:1.0:1767320673.116660:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320674.140632:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320675.165191:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320676.187399:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320676.524521:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320676.524564:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320676.524658:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630400 00000100:00100000:0.0:1767320676.524667:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630400 00000100:00100000:0.0:1767320676.524711:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6089 00000100:00100000:0.0:1767320676.524719:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920630400:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320676.524741:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320676.524806:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920630400:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 86us (293us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320676.524814:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6089 00000100:00100000:0.0:1767320676.524830:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630528 00000100:00100000:0.0:1767320676.524833:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630528 00000100:00100000:0.0:1767320676.524880:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5363 00000100:00100000:0.0:1767320676.524887:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920630528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320676.524921:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920630528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 35us (356us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320676.524927:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5363 00000100:00080000:0.0:1767320676.524943:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320677.084966:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985486080:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320677.085039:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320677.085233:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486080 00000100:00100000:3.0:1767320677.085241:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486080 00000100:00100000:3.0:1767320677.085302:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2688 00000100:00100000:3.0:1767320677.085312:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985486080:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320677.085340:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320677.085381:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985486080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (342us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320677.085389:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2688 00000100:00100000:0.0:1767320677.085951:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320677.085965:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985486208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320677.086003:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320677.086051:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320677.086064:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486208 00000100:00100000:0.0:1767320677.086068:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486208 00000100:00100000:0.0:1767320677.086100:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2689 00000100:00100000:0.0:1767320677.086107:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985486208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320677.086127:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320677.086158:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985486208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (153us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320677.086183:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2689 00000100:00080000:0.0:1767320677.086211:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320677.086256:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985486208:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320677.087361:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985486080:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320677.211402:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320677.597747:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320677.597756:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320677.597759:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320677.597763:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320677.597767:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320677.597775:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5920 00000100:00080000:0.0:1767320677.597780:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320677.597786:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320677.597826:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985486336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320677.597873:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320677.597877:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320677.597879:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320677.597882:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5920 00000100:00080000:0.0:1767320677.597885:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320677.597890:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320677.597903:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6a00 x1853163985486464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320677.597934:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320677.597938:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320677.597940:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320677.597943:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5920 00000100:00080000:0.0:1767320677.597946:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320677.597949:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320677.597959:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4e00 x1853163985486720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320677.597982:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320677.597984:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320677.597986:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320677.597989:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5920 00000100:00080000:0.0:1767320677.597991:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320677.597994:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320677.598004:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5500 x1853163985486976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320677.598568:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985486336:0@lo:38:kworker.0 00000100:00100000:1.0:1767320677.598637:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320677.598688:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985486464:0@lo:38:kworker.0 00000100:00100000:1.0:1767320677.598734:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320677.598763:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985486720:0@lo:38:kworker.0 00000100:00100000:1.0:1767320677.598790:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320677.598808:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985486976:0@lo:38:kworker.0 00000100:00100000:1.0:1767320677.598827:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320677.598874:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486464 00000100:00100000:1.0:1767320677.598881:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486464 00000100:00100000:1.0:1767320677.598911:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2013 00000100:00100000:1.0:1767320677.598919:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985486464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320677.598938:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320677.598979:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985486464:12345-0@lo:38:kworker.0 Request processed in 60us (244us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320677.598986:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2013 00000100:00100000:1.0:1767320677.599007:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486336 00000100:00100000:1.0:1767320677.599011:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486336 00000100:00100000:1.0:1767320677.599046:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486720 00000100:00100000:1.0:1767320677.599055:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486720 00000100:00100000:1.0:1767320677.599069:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486976 00000100:00100000:1.0:1767320677.599072:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486976 00000100:00100000:1.0:1767320677.599087:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6090 00000100:00100000:1.0:1767320677.599091:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985486336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320677.599101:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320677.599126:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985486336:12345-0@lo:38:kworker.0 Request processed in 33us (488us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320677.599132:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6090 00000100:00100000:1.0:1767320677.599142:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6091 00000100:00100000:1.0:1767320677.599146:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985486720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320677.599155:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320677.599217:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985486720:12345-0@lo:38:kworker.0 Request processed in 69us (432us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320677.599222:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6091 00000100:00100000:1.0:1767320677.599232:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6092 00000100:00100000:1.0:1767320677.599236:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985486976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320677.599245:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320677.599265:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985486976:12345-0@lo:38:kworker.0 Request processed in 27us (437us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320677.599270:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6092 00000100:00100000:3.0:1767320677.599332:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985486592:0@lo:400:kworker.0 00000100:00100000:3.0:1767320677.599367:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320677.599391:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486592 00000100:00100000:3.0:1767320677.599394:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486592 00000100:00100000:1.0:1767320677.599729:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320677.599738:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985486848:0@lo:400:kworker.0 00000100:00100000:1.0:1767320677.599761:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320677.599782:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985486848 00000100:00100000:1.0:1767320677.599785:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985486848 00000100:00080000:0.0:1767320677.599884:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320677.599889:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320677.599893:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985486336:0@lo:38:kworker.0 00000100:00080000:0.0:1767320677.599901:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320677.599904:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320677.599907:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985486464:0@lo:38:kworker.0 00000100:00080000:0.0:1767320677.599915:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320677.599918:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320677.599921:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985486720:0@lo:38:kworker.0 00000100:00080000:0.0:1767320677.599928:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320677.599930:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320677.599933:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985486976:0@lo:38:kworker.0 00000100:00080000:2.0:1767320677.601741:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320677.605911:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5364 00000100:00100000:2.0:1767320677.605931:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985486592:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320677.606026:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985486592:12345-0@lo:400:kworker.0 Request processed in 102us (6656us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320677.606033:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5364 00000100:00100000:2.0:1767320677.606098:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985486592:0@lo:400:kworker.0 00000100:00100000:1.0:1767320677.614596:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5365 00000100:00100000:1.0:1767320677.614621:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985486848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320677.614725:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985486848:12345-0@lo:400:kworker.0 Request processed in 116us (14961us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320677.614733:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5365 00000100:00100000:0.0:1767320677.614823:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985486848:0@lo:400:kworker.0 00010000:00010000:1.0:1767320678.235398:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320679.261272:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320680.284596:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320681.311038:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320681.642754:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320681.642824:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630656 00000100:00100000:2.0:1767320681.642829:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320681.642836:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630656 00000100:00100000:1.0:1767320681.642872:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630784 00000100:00100000:2.0:1767320681.642883:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6093 00000100:00100000:1.0:1767320681.642886:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630784 00000100:00100000:2.0:1767320681.642891:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920630656:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320681.642914:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320681.642949:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5366 00000100:00100000:1.0:1767320681.642956:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920630784:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320681.642985:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920630656:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 93us (240us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320681.642992:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6093 00000100:00100000:1.0:1767320681.642993:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920630784:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 39us (163us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320681.642998:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5366 00000100:00080000:0.0:1767320681.644583:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320682.206238:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985487104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320682.206361:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320682.206517:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985487104 00000100:00100000:2.0:1767320682.206526:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985487104 00000100:00100000:1.0:1767320682.206593:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985487232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320682.206625:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320682.206673:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320682.212637:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985487232 00000100:00100000:2.0:1767320682.212654:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985487232 00000100:00100000:2.0:1767320682.212732:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2690 00000100:00100000:2.0:1767320682.212744:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb1c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985487104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320682.212789:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320682.212868:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb1c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985487104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 127us (6506us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320682.212876:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2690 00000100:00100000:2.0:1767320682.213070:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2691 00000100:00100000:2.0:1767320682.213077:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985487232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320682.213092:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320682.213121:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985487232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (6496us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320682.213128:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2691 00000100:00080000:1.0:1767320682.214667:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320682.214732:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985487232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320682.214805:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985487104:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320682.331401:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320682.715289:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320682.715302:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320682.715305:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320682.715309:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320682.715314:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320682.715322:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5925 00000100:00080000:2.0:1767320682.715327:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320682.715333:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320682.715373:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7ed80 x1853163985487360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320682.715401:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320682.715405:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320682.715406:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320682.715409:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5925 00000100:00080000:2.0:1767320682.715412:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320682.715416:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320682.715430:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ed80 x1853163985487488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320682.715450:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985487360:0@lo:38:kworker.0 00000100:00100000:1.0:1767320682.715517:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985487616:0@lo:400:kworker.0 00000100:00100000:0.0:1767320682.715529:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320682.715612:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985487360 00000100:00100000:3.0:1767320682.715620:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985487360 00000100:00100000:3.0:1767320682.715666:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6094 00000100:00100000:3.0:1767320682.715674:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985487360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320682.715694:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320682.715937:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320682.715941:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320682.715943:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320682.715948:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5925 00000100:00080000:2.0:1767320682.715951:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320682.715955:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320682.715971:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c000 x1853163985487744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320682.716028:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985487872:0@lo:400:kworker.0 00000100:00080000:2.0:1767320682.716060:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320682.716064:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320682.716066:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320682.716069:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5925 00000100:00080000:2.0:1767320682.716071:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320682.716075:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320682.716088:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18e680 x1853163985488000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320682.720538:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985487360:12345-0@lo:38:kworker.0 Request processed in 4852us (4996us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320682.720554:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6094 00000100:00100000:1.0:1767320682.720794:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320682.721023:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985487872 00000100:00100000:3.0:1767320682.721031:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985487872 00000100:00100000:3.0:1767320682.721126:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5367 00000100:00100000:3.0:1767320682.721136:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985487872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320682.721207:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985487872:12345-0@lo:400:kworker.0 Request processed in 73us (413us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320682.721213:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5367 00000100:00100000:3.0:1767320682.721400:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985487872:0@lo:400:kworker.0 00000100:00100000:2.0:1767320682.721479:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320682.721693:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320682.721896:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985487616 00000100:00100000:3.0:1767320682.721902:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985487616 00000100:00100000:3.0:1767320682.721982:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5368 00000100:00100000:3.0:1767320682.721990:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985487616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320682.722043:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985487616:12345-0@lo:400:kworker.0 Request processed in 54us (564us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320682.722049:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5368 00000100:00100000:2.0:1767320682.722157:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985487616:0@lo:400:kworker.0 00000100:00080000:0.0:1767320682.724082:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320682.724093:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320682.724098:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985487360:0@lo:38:kworker.0 00000100:00100000:0.0:1767320682.724109:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985487488:0@lo:38:kworker.0 00000100:00100000:0.0:1767320682.724158:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320682.724226:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985487744:0@lo:38:kworker.0 00000100:00100000:0.0:1767320682.724248:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320682.724270:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985488000:0@lo:38:kworker.0 00000100:00100000:0.0:1767320682.724288:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320682.724326:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985487744 00000100:00100000:0.0:1767320682.724332:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985487744 00000100:00100000:0.0:1767320682.724372:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488000 00000100:00100000:0.0:1767320682.724375:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488000 00000100:00100000:2.0:1767320682.724391:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985487488 00000100:00100000:0.0:1767320682.724394:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6095 00000100:00100000:2.0:1767320682.724395:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985487488 00000100:00100000:0.0:1767320682.724400:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985487744:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320682.724414:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2014 00000100:00100000:2.0:1767320682.724418:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985487488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320682.724420:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767320682.724426:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320682.724458:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985487744:12345-0@lo:38:kworker.0 Request processed in 56us (207us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320682.724458:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985487488:12345-0@lo:38:kworker.0 Request processed in 37us (299us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320682.724466:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6095 00000100:00100000:2.0:1767320682.724466:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2014 00000100:00100000:0.0:1767320682.724480:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6096 00000100:00100000:0.0:1767320682.724484:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985488000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320682.724492:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320682.724607:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985488000:12345-0@lo:38:kworker.0 Request processed in 120us (317us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320682.724613:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6096 00000100:00080000:0.0:1767320682.724638:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320682.724642:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320682.724646:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985487744:0@lo:38:kworker.0 00000100:00080000:0.0:1767320682.724718:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320682.724722:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320682.724725:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985488000:0@lo:38:kworker.0 00000100:00080000:0.0:1767320682.724735:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320682.724738:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320682.724740:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985487488:0@lo:38:kworker.0 00000100:00080000:1.0:1767320682.726728:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320683.355912:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320684.379600:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320685.410494:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320686.428378:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320686.761858:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320686.761954:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320686.762205:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920631040 00000100:00100000:3.0:1767320686.762218:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920631040 00000100:00100000:3.0:1767320686.762292:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920630912 00000100:00100000:3.0:1767320686.762295:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920630912 00000100:00100000:3.0:1767320686.762321:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5369 00000100:00100000:3.0:1767320686.762331:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920631040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320686.762410:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920631040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (562us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320686.762418:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5369 00000100:00100000:3.0:1767320686.762436:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6097 00000100:00100000:3.0:1767320686.762441:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920630912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320686.762461:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320686.762488:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920630912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (534us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320686.762494:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6097 00000100:00080000:1.0:1767320686.762935:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320687.323549:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985488128:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320687.323628:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320687.323790:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488128 00000100:00100000:2.0:1767320687.323798:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488128 00000100:00100000:2.0:1767320687.323855:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2692 00000100:00100000:2.0:1767320687.323864:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985488128:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320687.323893:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320687.324009:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320687.324019:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985488256:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320687.324043:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320687.324079:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488256 00000100:00100000:3.0:1767320687.324083:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488256 00000100:00100000:3.0:1767320687.324105:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2693 00000100:00100000:3.0:1767320687.324109:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985488256:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320687.324122:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320687.324144:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985488256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 35us (100us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320687.324149:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2693 00000100:00100000:3.0:1767320687.324203:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985488256:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320687.324235:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320687.324239:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320687.328795:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985488128:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4919us (5152us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320687.328811:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2692 00000100:00100000:0.0:1767320687.331530:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985488128:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320687.452132:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320687.835881:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320687.835978:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320687.835981:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320687.835986:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320687.836001:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320687.836011:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5930 00000100:00080000:0.0:1767320687.836016:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320687.836023:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320687.836062:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7b80 x1853163985488384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320687.836091:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320687.836094:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320687.836096:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320687.836099:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5930 00000100:00080000:0.0:1767320687.836102:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320687.836105:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320687.836118:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5c00 x1853163985488512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320687.836178:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985488384:0@lo:38:kworker.0 00000100:00080000:0.0:1767320687.836183:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320687.836188:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320687.836190:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320687.836194:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5930 00000100:00080000:0.0:1767320687.836197:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320687.836201:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320687.836214:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4000 x1853163985488768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320687.836238:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320687.836241:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320687.836243:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320687.836248:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320687.836251:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5930 00000100:00080000:0.0:1767320687.836255:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320687.836258:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320687.836270:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5f80 x1853163985489024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320687.836311:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985488512:0@lo:38:kworker.0 00000100:00100000:2.0:1767320687.836350:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320687.836378:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985488768:0@lo:38:kworker.0 00000100:00100000:2.0:1767320687.836399:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320687.836417:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985489024:0@lo:38:kworker.0 00000100:00100000:2.0:1767320687.836437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320687.836468:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488384 00000100:00100000:2.0:1767320687.836475:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488384 00000100:00100000:2.0:1767320687.836502:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488768 00000100:00100000:2.0:1767320687.836506:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488768 00000100:00100000:2.0:1767320687.836517:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489024 00000100:00100000:2.0:1767320687.836519:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489024 00000100:00100000:2.0:1767320687.836535:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6098 00000100:00100000:2.0:1767320687.836541:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985488384:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320687.836601:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6099 00000100:00100000:3.0:1767320687.836609:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985488768:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320687.836696:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6100 00000100:00100000:1.0:1767320687.836703:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985489024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320687.836732:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320687.836774:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985488384:12345-0@lo:38:kworker.0 Request processed in 232us (529us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320687.836782:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6098 00000100:00100000:2.0:1767320687.836808:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488512 00000100:00100000:2.0:1767320687.836811:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488512 00000100:00100000:2.0:1767320687.836842:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2015 00000100:00100000:2.0:1767320687.836847:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985488512:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320687.836911:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985489152:0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.836943:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320687.836966:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320687.836972:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985488896:0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.836992:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320687.837030:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488896 00000100:00100000:0.0:1767320687.837034:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488896 00000100:00100000:0.0:1767320687.837070:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5370 00000100:00100000:0.0:1767320687.837077:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985488896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837117:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985488896:12345-0@lo:400:kworker.0 Request processed in 41us (124us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320687.837123:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5370 00000100:00100000:0.0:1767320687.837145:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985489280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837193:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320687.837214:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320687.837219:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985488640:0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837240:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320687.837262:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985488640 00000100:00100000:0.0:1767320687.837265:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985488640 00000100:00100000:0.0:1767320687.837284:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5371 00000100:00100000:0.0:1767320687.837290:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985488640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837339:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985488640:12345-0@lo:400:kworker.0 Request processed in 48us (97us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320687.837345:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5371 00000100:00100000:0.0:1767320687.837378:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985488640:0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837407:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985488896:0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837431:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489152 00000100:00100000:0.0:1767320687.837434:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489152 00000100:00100000:0.0:1767320687.837456:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489280 00000100:00100000:0.0:1767320687.837459:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489280 00000100:00100000:0.0:1767320687.837476:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3485 00000100:00100000:0.0:1767320687.837482:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985489152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837515:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985489152:12345-0@lo:400:kworker.0 Request processed in 32us (571us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320687.837521:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3485 00000100:00100000:0.0:1767320687.837531:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3486 00000100:00100000:0.0:1767320687.837537:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985489280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837566:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985489280:12345-0@lo:400:kworker.0 Request processed in 29us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320687.837571:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3486 00000100:00100000:0.0:1767320687.837598:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985489280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320687.837618:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985489152:0@lo:400:kworker.0 00000100:00080000:0.0:1767320687.837706:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320687.837712:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320687.837715:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985488384:0@lo:38:kworker.0 00000100:00080000:0.0:1767320687.837765:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:3.0:1767320687.846660:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767320687.846691:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320687.846773:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985488768:12345-0@lo:38:kworker.0 Request processed in 10161us (10370us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320687.846784:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6099 00010000:00020000:1.0:1767320687.846915:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320687.847019:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985488512:12345-0@lo:38:kworker.0 Request processed in 10169us (10667us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320687.847029:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2015 00000100:00080000:0.0:1767320687.847099:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320687.847106:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320687.847110:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985488768:0@lo:38:kworker.0 00000100:00100000:1.0:1767320687.847263:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985489024:12345-0@lo:38:kworker.0 Request processed in 10558us (10824us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320687.847270:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6100 00000100:00080000:0.0:1767320687.847480:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320687.847485:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320687.847488:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985489024:0@lo:38:kworker.0 00000100:00080000:0.0:1767320687.847499:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320687.847502:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320687.847505:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985488512:0@lo:38:kworker.0 00010000:00010000:2.0:1767320688.477592:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320689.500220:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320690.523356:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320691.548988:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320691.889109:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320691.889260:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767320691.891868:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920631424 offset 0 length 520: 4 00000100:00100000:2.0:1767320691.892481:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920631296 00000100:00100000:2.0:1767320691.892491:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920631296 00000100:00100000:2.0:1767320691.894740:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920631168 00000100:00100000:2.0:1767320691.894756:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920631168 00000100:00100000:2.0:1767320691.894786:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5372 00000100:00100000:2.0:1767320691.894801:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920631296:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320691.895064:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6101 00000100:00100000:1.0:1767320691.895071:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920631168:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320691.895092:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320691.895195:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320691.895674:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920631296:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 875us (6572us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320691.895684:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5372 00000100:00100000:1.0:1767320691.899740:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920631168:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 4654us (10466us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320691.899756:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6101 00000100:00100000:0.0:1767320692.443497:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985489408:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320692.443534:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320692.443550:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985489536:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320692.443671:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320692.443858:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320692.443892:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489408 00000100:00100000:0.0:1767320692.443900:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489408 00000100:00100000:0.0:1767320692.443948:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489536 00000100:00100000:0.0:1767320692.443951:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489536 00000100:00100000:0.0:1767320692.443972:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2694 00000100:00100000:0.0:1767320692.443980:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985489408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320692.444105:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320692.444145:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985489408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 166us (475us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320692.444153:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2694 00000100:00100000:0.0:1767320692.444188:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2695 00000100:00100000:0.0:1767320692.444194:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985489536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320692.444208:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320692.444232:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985489536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 38us (486us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320692.444237:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2695 00000100:00080000:0.0:1767320692.444250:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320692.444253:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320692.444282:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985489536:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320692.444307:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985489408:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320692.571476:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320692.955288:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320667/real 1767320667] req@ffff8ce19e9c4000 x1853163985484160/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320692 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320692.955296:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320692.955307:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320692.955310:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320692.955313:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320692.955316:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320692.955321:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5936 00000100:00080000:3.0:1767320692.955325:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320692.955326:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19e9c4000 x1853163985484160/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320692 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320692.955332:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00080000:2.0:1767320692.955346:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320692.955365:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320692.955368:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985484160:0@lo:250:kworker.0 00000100:00100000:3.0:1767320692.955373:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05c00 x1853163985489664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320692.955454:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320692.955457:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320692.955459:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320692.955461:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5936 00000100:00080000:3.0:1767320692.955463:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320692.955466:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320692.955477:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04700 x1853163985489792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320692.955525:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985489920:0@lo:400:kworker.0 00000100:00100000:0.0:1767320692.955600:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320692.955661:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489920 00000100:00080000:3.0:1767320692.955669:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320692.955671:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489920 00000100:00080000:3.0:1767320692.955676:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320692.955678:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320692.955681:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5936 00000100:00080000:3.0:1767320692.955683:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320692.955686:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320692.955697:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985490048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320692.955720:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320692.955723:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320692.955725:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320692.955727:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5936 00000100:00080000:3.0:1767320692.955729:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320692.955731:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320692.955738:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05880 x1853163985490304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320692.955746:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5373 00000100:00100000:1.0:1767320692.955761:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985489920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.955815:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985489664:0@lo:38:kworker.0 00000100:00100000:2.0:1767320692.955844:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320692.955872:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985489792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320692.955918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320692.955940:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985490048:0@lo:38:kworker.0 00000100:00100000:2.0:1767320692.955960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320692.955977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985490304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320692.956010:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320692.956054:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489664 00000100:00100000:2.0:1767320692.956058:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489664 00000100:00100000:2.0:1767320692.956080:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985490048 00000100:00100000:2.0:1767320692.956083:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985490048 00000100:00100000:2.0:1767320692.956092:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985490304 00000100:00100000:2.0:1767320692.956094:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985490304 00000100:00100000:2.0:1767320692.956104:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6102 00000100:00100000:2.0:1767320692.956109:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985489664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320692.956127:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320692.956193:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985489664:12345-0@lo:38:kworker.0 Request processed in 82us (348us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320692.956201:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6102 00000100:00100000:2.0:1767320692.956219:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6103 00000100:00100000:2.0:1767320692.956223:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985490048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320692.956232:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320692.956256:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985490048:12345-0@lo:38:kworker.0 Request processed in 32us (295us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320692.956262:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6103 00000100:00100000:2.0:1767320692.956273:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6104 00000100:00100000:2.0:1767320692.956277:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985490304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320692.956285:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320692.956307:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985490304:12345-0@lo:38:kworker.0 Request processed in 29us (297us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320692.956313:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6104 00000100:00100000:2.0:1767320692.956332:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985489792 00000100:00100000:2.0:1767320692.956336:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985489792 00000100:00100000:2.0:1767320692.956356:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2016 00000100:00100000:2.0:1767320692.956361:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985489792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320692.956370:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320692.956394:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985489792:12345-0@lo:38:kworker.0 Request processed in 32us (476us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320692.956399:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2016 00000100:00080000:2.0:1767320692.956434:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320692.956438:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320692.956441:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985489664:0@lo:38:kworker.0 00000100:00080000:2.0:1767320692.956449:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320692.956451:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320692.956454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985489792:0@lo:38:kworker.0 00000100:00080000:2.0:1767320692.956460:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320692.956463:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320692.956465:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985490048:0@lo:38:kworker.0 00000100:00080000:2.0:1767320692.956471:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320692.956473:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320692.956476:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985490304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320692.956524:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985490560:0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.956550:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320692.956591:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985490176:0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.956612:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320692.956633:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320692.956638:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985490432:0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.956658:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320692.956680:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985490176 00000100:00100000:2.0:1767320692.956684:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985490176 00000100:00100000:2.0:1767320692.956715:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5374 00000100:00100000:2.0:1767320692.956721:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985490176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.956754:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985490176:12345-0@lo:400:kworker.0 Request processed in 32us (140us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320692.956760:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5374 00000100:00100000:2.0:1767320692.956787:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985490176:0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.956832:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985490560 00000100:00100000:2.0:1767320692.956835:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985490560 00000100:00100000:2.0:1767320692.956859:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985490432 00000100:00100000:2.0:1767320692.956862:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985490432 00000100:00100000:2.0:1767320692.956878:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3487 00000100:00100000:2.0:1767320692.956883:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985490560:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.956915:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985490560:12345-0@lo:400:kworker.0 Request processed in 32us (365us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320692.956922:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3487 00000100:00100000:2.0:1767320692.956933:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3488 00000100:00100000:2.0:1767320692.956939:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985490432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.956966:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985490432:12345-0@lo:400:kworker.0 Request processed in 28us (307us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320692.956973:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3488 00000100:00100000:2.0:1767320692.957004:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985490560:0@lo:400:kworker.0 00000100:00100000:2.0:1767320692.957026:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985490432:0@lo:400:kworker.0 00000100:00080000:2.0:1767320692.957073:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320692.961429:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985489920:12345-0@lo:400:kworker.0 Request processed in 5661us (5816us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320692.961455:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5373 00000100:00100000:0.0:1767320692.961490:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985489920:0@lo:400:kworker.0 00000100:00080000:3.0:1767320692.966293:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320692.966301:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320692.966305:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320692.966312:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5915 00000100:00080000:3.0:1767320692.966316:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320692.966322:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320692.966355:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffea00 x1853163985490688/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320692.966476:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985490688:0@lo:250:kworker.0 00000400:00000100:2.0:1767320692.966538:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985490688 offset 0 length 520: 4 00010000:00010000:2.0:1767320693.596010:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320694.619397:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320695.644080:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320696.667800:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320697.001564:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320697.001745:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920631680 00000100:00100000:1.0:1767320697.001758:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920631680 00000100:00100000:1.0:1767320697.001847:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5375 00000100:00100000:1.0:1767320697.001855:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920631680:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320697.001931:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920631680:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320697.001939:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5375 00000100:00080000:2.0:1767320697.002048:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320697.002358:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320697.002442:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920631552 00000100:00100000:0.0:1767320697.002447:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920631552 00000100:00100000:0.0:1767320697.002483:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6105 00000100:00100000:0.0:1767320697.002498:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920631552:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320697.002518:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320697.002564:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920631552:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 67us (207us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320697.002571:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6105 00000100:00100000:2.0:1767320697.563924:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320697.563949:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985490816:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320697.564018:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320697.564083:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985490816 00000100:00100000:2.0:1767320697.564091:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985490816 00000100:00100000:2.0:1767320697.564148:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2696 00000100:00100000:2.0:1767320697.564159:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985490816:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320697.564212:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320697.564255:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985490816:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320697.564262:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2696 00000100:00080000:2.0:1767320697.564285:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320697.564318:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985490816:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320697.564645:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320697.564657:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985490944:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320697.564687:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320697.564774:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985490944 00000100:00100000:1.0:1767320697.564779:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985490944 00000100:00100000:1.0:1767320697.564826:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2697 00000100:00100000:1.0:1767320697.564833:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985490944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320697.564849:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320697.564889:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985490944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (198us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320697.564889:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320697.564904:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2697 00000100:00100000:0.0:1767320697.564994:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985490944:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320697.691590:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320698.075350:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320698.075364:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320698.075368:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320698.075373:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320698.075378:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320698.075389:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5941 00000100:00080000:3.0:1767320698.075395:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320698.075403:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320698.075455:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9dc00 x1853163985491072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320698.075525:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320698.075530:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320698.075534:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320698.075538:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5941 00000100:00080000:3.0:1767320698.075540:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320698.075544:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320698.075558:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9ca80 x1853163985491200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320698.077574:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985491072:0@lo:38:kworker.0 00000100:00100000:2.0:1767320698.077653:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320698.077721:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985491200:0@lo:38:kworker.0 00000100:00100000:2.0:1767320698.077751:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320698.077763:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320698.077773:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320698.077778:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320698.077787:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5941 00000100:00080000:3.0:1767320698.077791:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320698.077800:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320698.077813:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491200 00000100:00100000:2.0:1767320698.077823:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491200 00000100:00100000:3.0:1767320698.077844:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7100 x1853163985491456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320698.077864:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2017 00000100:00100000:2.0:1767320698.077874:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985491200:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320698.077900:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320698.077911:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320698.077916:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320698.077919:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320698.077923:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5941 00000100:00080000:3.0:1767320698.077926:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320698.077932:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320698.077949:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985491200:12345-0@lo:38:kworker.0 Request processed in 73us (195us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320698.077951:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb100 x1853163985491712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320698.077979:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2017 00000100:00100000:2.0:1767320698.078021:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985491584:0@lo:400:kworker.0 00000100:00100000:2.0:1767320698.078054:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320698.078064:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985491456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320698.078084:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320698.078095:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985491840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320698.078133:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320698.078142:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320698.078211:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985491712:0@lo:38:kworker.0 00000100:00100000:2.0:1767320698.078225:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491072 00000100:00100000:2.0:1767320698.078232:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491072 00000100:00100000:3.0:1767320698.078250:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320698.078301:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320698.078307:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320698.078312:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985491200:0@lo:38:kworker.0 00000100:00100000:0.0:1767320698.078323:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985491328:0@lo:400:kworker.0 00000100:00100000:0.0:1767320698.078364:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320698.078375:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491840 00000100:00100000:3.0:1767320698.078382:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491840 00000100:00100000:3.0:1767320698.078441:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3489 00000100:00100000:3.0:1767320698.078451:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985491840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320698.078526:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985491840:12345-0@lo:400:kworker.0 Request processed in 76us (388us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320698.078534:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3489 00000100:00100000:2.0:1767320698.080518:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491584 00000100:00100000:2.0:1767320698.080534:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491584 00000100:00100000:2.0:1767320698.080616:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491456 00000100:00100000:2.0:1767320698.080620:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491456 00000100:00100000:2.0:1767320698.080630:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491712 00000100:00100000:2.0:1767320698.080633:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491712 00000100:00100000:2.0:1767320698.080645:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5376 00000100:00100000:2.0:1767320698.080658:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985491584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320698.080740:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985491584:12345-0@lo:400:kworker.0 Request processed in 86us (2688us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320698.080747:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5376 00000100:00100000:2.0:1767320698.080765:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6106 00000100:00100000:2.0:1767320698.080770:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985491072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320698.080795:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320698.080833:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985491072:12345-0@lo:38:kworker.0 Request processed in 61us (3180us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320698.080838:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6106 00000100:00100000:2.0:1767320698.080850:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6107 00000100:00100000:2.0:1767320698.080854:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985491456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320698.080863:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320698.080887:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985491456:12345-0@lo:38:kworker.0 Request processed in 31us (2756us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320698.080893:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6107 00000100:00100000:2.0:1767320698.080904:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6108 00000100:00100000:2.0:1767320698.080907:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985491712:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320698.080916:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320698.081339:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320698.081350:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985491968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320698.081384:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320698.081433:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491328 00000100:00100000:0.0:1767320698.081438:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491328 00000100:00100000:0.0:1767320698.081472:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5377 00000100:00100000:0.0:1767320698.081479:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985491328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320698.081516:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985491328:12345-0@lo:400:kworker.0 Request processed in 37us (3152us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320698.081522:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5377 00000100:00100000:0.0:1767320698.081563:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985491328:0@lo:400:kworker.0 00000100:00100000:0.0:1767320698.081587:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985491968 00000100:00100000:0.0:1767320698.081590:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985491968 00000100:00100000:0.0:1767320698.081616:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3490 00000100:00100000:0.0:1767320698.081624:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985491968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320698.081659:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985491968:12345-0@lo:400:kworker.0 Request processed in 37us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320698.081665:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3490 00000100:00100000:0.0:1767320698.081691:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985491968:0@lo:400:kworker.0 00000100:00080000:0.0:1767320698.081736:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320698.081755:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985491840:0@lo:400:kworker.0 00000100:00100000:0.0:1767320698.081765:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985491584:0@lo:400:kworker.0 00000100:00080000:0.0:1767320698.081812:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320698.081818:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320698.081821:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985491456:0@lo:38:kworker.0 00000100:00080000:0.0:1767320698.081831:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320698.081834:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320698.081837:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985491072:0@lo:38:kworker.0 00000100:00100000:2.0:1767320698.084724:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985491712:12345-0@lo:38:kworker.0 Request processed in 3805us (6466us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320698.084737:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6108 00000100:00080000:0.0:1767320698.085063:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320698.085069:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320698.085073:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985491712:0@lo:38:kworker.0 00010000:00010000:1.0:1767320698.715610:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320699.739668:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320700.764493:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320701.787626:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320702.124266:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320702.124308:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320702.124451:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920631936 00000100:00100000:2.0:1767320702.124470:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920631936 00000100:00080000:0.0:1767320702.124552:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320702.124569:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920631808 00000100:00100000:2.0:1767320702.124573:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920631808 00000100:00100000:2.0:1767320702.124596:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5378 00000100:00100000:2.0:1767320702.124608:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920631936:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320702.124629:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6109 00000100:00100000:0.0:1767320702.124641:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920631808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320702.124667:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320702.124712:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920631936:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (452us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320702.124723:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5378 00000100:00100000:0.0:1767320702.124739:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920631808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 93us (422us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320702.124752:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6109 00000100:00100000:3.0:1767320702.683585:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320702.683619:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985492096:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320702.683721:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320702.684011:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492096 00000100:00100000:2.0:1767320702.684023:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492096 00000100:00100000:2.0:1767320702.684233:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2698 00000100:00100000:2.0:1767320702.684248:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985492096:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320702.684299:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320702.684400:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985492096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 156us (678us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320702.684410:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2698 00000100:00100000:0.0:1767320702.684504:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985492096:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320702.685511:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320702.685531:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985492224:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320702.685597:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320702.685750:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492224 00000100:00100000:1.0:1767320702.685760:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492224 00000100:00080000:3.0:1767320702.685829:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320702.685835:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320702.686001:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2699 00000100:00100000:1.0:1767320702.686011:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985492224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320702.686047:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320702.686121:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985492224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (524us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320702.686130:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2699 00000100:00100000:0.0:1767320702.686288:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985492224:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320702.811496:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320703.195621:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320703.195633:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320703.195637:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320703.195643:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320703.195649:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320703.195659:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5946 00000100:00080000:3.0:1767320703.195664:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320703.195672:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320703.195713:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552f480 x1853163985492352/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320703.195749:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320703.195752:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320703.195754:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320703.195757:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5946 00000100:00080000:3.0:1767320703.195759:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320703.195762:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320703.195773:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552d880 x1853163985492480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320703.195810:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320703.195813:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320703.195815:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320703.195818:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5946 00000100:00080000:3.0:1767320703.195822:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320703.195826:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320703.195842:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6a00 x1853163985492736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320703.195853:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985492608:0@lo:400:kworker.0 00000100:00080000:3.0:1767320703.195882:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320703.195887:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320703.195890:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320703.195894:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5946 00000100:00080000:3.0:1767320703.195896:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320703.195901:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320703.195918:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351ad80 x1853163985492992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320703.195970:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320703.195992:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985492864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320703.196062:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320703.196075:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320703.196079:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985493248:0@lo:400:kworker.0 00000100:00100000:3.0:1767320703.196124:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320703.196133:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985493120:0@lo:400:kworker.0 00000100:00100000:3.0:1767320703.196161:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320703.196235:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492608 00000100:00100000:3.0:1767320703.196242:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492608 00000100:00100000:3.0:1767320703.196288:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492864 00000100:00100000:3.0:1767320703.196291:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492864 00000100:00100000:3.0:1767320703.196315:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5379 00000100:00100000:3.0:1767320703.196323:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985492608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320703.196438:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5380 00000100:00100000:3.0:1767320703.196440:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985492608:12345-0@lo:400:kworker.0 Request processed in 114us (468us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320703.196452:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5379 00000100:00100000:2.0:1767320703.196460:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985492864:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320703.196486:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320703.196540:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985492864:12345-0@lo:400:kworker.0 Request processed in 89us (473us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320703.196551:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5380 00000100:00100000:3.0:1767320703.196577:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985492864:0@lo:400:kworker.0 00000100:00100000:2.0:1767320703.196609:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985493120 00000100:00100000:2.0:1767320703.196618:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985493120 00000100:00100000:2.0:1767320703.196684:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3491 00000100:00100000:2.0:1767320703.196699:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985493120:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320703.196700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985492352:0@lo:38:kworker.0 00000100:00100000:1.0:1767320703.196750:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320703.196784:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985492480:0@lo:38:kworker.0 00000100:00100000:1.0:1767320703.196806:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320703.196827:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985492736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320703.196845:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320703.196860:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985492992:0@lo:38:kworker.0 00000100:00100000:1.0:1767320703.196878:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320703.196909:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492352 00000100:00100000:1.0:1767320703.196914:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492352 00000100:00100000:1.0:1767320703.196937:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492736 00000100:00100000:1.0:1767320703.196939:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492736 00000100:00100000:1.0:1767320703.196951:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492992 00000100:00100000:1.0:1767320703.196953:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492992 00000100:00100000:1.0:1767320703.196964:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6110 00000100:00100000:1.0:1767320703.196969:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985492352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320703.196987:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320703.197018:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985492352:12345-0@lo:38:kworker.0 Request processed in 48us (269us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320703.197024:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6110 00000100:00100000:1.0:1767320703.197037:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6111 00000100:00100000:1.0:1767320703.197041:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985492736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320703.197048:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320703.197070:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985492736:12345-0@lo:38:kworker.0 Request processed in 28us (224us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320703.197075:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6111 00000100:00100000:1.0:1767320703.197084:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6112 00000100:00100000:1.0:1767320703.197088:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985492992:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320703.197096:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320703.197116:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985492992:12345-0@lo:38:kworker.0 Request processed in 26us (237us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320703.197122:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6112 00000100:00080000:1.0:1767320703.197148:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320703.197153:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320703.197156:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985492352:0@lo:38:kworker.0 00000100:00080000:1.0:1767320703.197197:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320703.197202:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320703.197205:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985492736:0@lo:38:kworker.0 00000100:00080000:1.0:1767320703.197224:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320703.197227:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320703.197232:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985492992:0@lo:38:kworker.0 00000100:00100000:0.0:1767320703.197246:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320703.197301:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985492608:0@lo:400:kworker.0 00000100:00100000:1.0:1767320703.197304:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985493248 00000100:00100000:1.0:1767320703.197309:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985493248 00000100:00100000:1.0:1767320703.197493:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3492 00000100:00100000:1.0:1767320703.197507:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985493248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320703.197514:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985492480 00000100:00100000:0.0:1767320703.197522:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985492480 00000100:00100000:1.0:1767320703.197561:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985493248:12345-0@lo:400:kworker.0 Request processed in 59us (318us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320703.197568:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3492 00000100:00100000:0.0:1767320703.197999:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2018 00000100:00100000:0.0:1767320703.198007:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985492480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320703.198028:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320703.198088:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985492480:12345-0@lo:38:kworker.0 Request processed in 80us (1280us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320703.198106:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2018 00000100:00100000:0.0:1767320703.198147:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985493248:0@lo:400:kworker.0 00000100:00080000:0.0:1767320703.198218:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320703.198225:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320703.198229:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985492480:0@lo:38:kworker.0 00000100:00100000:2.0:1767320703.198588:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985493120:12345-0@lo:400:kworker.0 Request processed in 1895us (2424us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320703.198598:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3491 00000100:00100000:3.0:1767320703.198731:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985493120:0@lo:400:kworker.0 00010000:00010000:1.0:1767320703.835556:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320704.859661:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320705.884955:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320706.908381:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320707.251765:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320707.251979:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320707.252105:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920632192 00000100:00100000:0.0:1767320707.252119:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920632192 00000100:00100000:1.0:1767320707.252309:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920632064 00000100:00100000:1.0:1767320707.252326:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920632064 00000100:00100000:0.0:1767320707.252381:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5381 00000100:00100000:0.0:1767320707.252400:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920632192:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320707.252414:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6113 00000100:00100000:1.0:1767320707.252424:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920632064:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320707.252450:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320707.252615:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320707.257084:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920632192:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 4670us (5316us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320707.257106:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5381 00000100:00100000:1.0:1767320707.259829:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920632064:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 7392us (7836us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320707.259845:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6113 00000100:00100000:0.0:1767320707.803511:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985493504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320707.803528:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320707.803546:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985493376:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320707.803588:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320707.803709:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320707.803737:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985493504 00000100:00100000:2.0:1767320707.803747:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985493504 00000100:00100000:0.0:1767320707.803778:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985493376 00000100:00100000:0.0:1767320707.803782:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985493376 00000100:00080000:3.0:1767320707.803837:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320707.803842:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320707.803908:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2700 00000100:00100000:2.0:1767320707.803920:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985493504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320707.803947:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320707.804002:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985493504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (414us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320707.804010:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2700 00000100:00100000:2.0:1767320707.804037:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2701 00000100:00100000:2.0:1767320707.804045:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985493376:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320707.804060:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320707.804091:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985493376:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (382us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320707.804097:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2701 00000100:00100000:3.0:1767320707.804189:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985493504:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320707.806902:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985493376:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320707.932109:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320708.315445:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320708.315457:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320708.315460:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320708.315464:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320708.315468:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320708.315476:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5951 00000100:00080000:0.0:1767320708.315481:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320708.315488:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320708.315533:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9f80 x1853163985493632/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320708.315561:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320708.315564:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320708.315566:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320708.315570:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5951 00000100:00080000:0.0:1767320708.315572:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320708.315575:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320708.315590:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985493760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320708.315635:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320708.315639:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320708.315641:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320708.315644:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5951 00000100:00080000:0.0:1767320708.315684:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320708.315688:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320708.315711:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6a00 x1853163985494016/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320708.315735:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320708.315738:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320708.315740:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320708.315743:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5951 00000100:00080000:0.0:1767320708.315744:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320708.315747:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320708.315756:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5c00 x1853163985494272/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320708.316042:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985494144:0@lo:400:kworker.0 00000100:00100000:3.0:1767320708.316113:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320708.316202:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320708.316209:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985493888:0@lo:400:kworker.0 00000100:00100000:3.0:1767320708.316229:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320708.316258:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494144 00000100:00100000:3.0:1767320708.316264:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494144 00000100:00100000:3.0:1767320708.316319:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985493888 00000100:00100000:3.0:1767320708.316321:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985493888 00000100:00100000:3.0:1767320708.316338:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5382 00000100:00100000:3.0:1767320708.316343:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985494144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320708.316385:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985494144:12345-0@lo:400:kworker.0 Request processed in 41us (272us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320708.316390:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5382 00000100:00100000:3.0:1767320708.316403:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5383 00000100:00100000:3.0:1767320708.316407:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985493888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320708.316429:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985493888:12345-0@lo:400:kworker.0 Request processed in 21us (199us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320708.316433:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5383 00000100:00080000:3.0:1767320708.316446:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320708.316473:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985494144:0@lo:400:kworker.0 00000100:00100000:3.0:1767320708.316499:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985493888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320708.316611:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985493632:0@lo:38:kworker.0 00000100:00100000:1.0:1767320708.316645:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320708.317476:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985493760:0@lo:38:kworker.0 00000100:00100000:1.0:1767320708.317514:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320708.317540:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985494016:0@lo:38:kworker.0 00000100:00100000:1.0:1767320708.317561:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320708.329301:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985494272:0@lo:38:kworker.0 00000100:00100000:1.0:1767320708.329363:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320708.331390:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985493632 00000100:00100000:2.0:1767320708.331405:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985493632 00000100:00100000:2.0:1767320708.331464:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494016 00000100:00100000:2.0:1767320708.331468:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494016 00000100:00100000:2.0:1767320708.331485:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6114 00000100:00100000:2.0:1767320708.331493:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985493632:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320708.331518:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320708.331593:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985493632:12345-0@lo:38:kworker.0 Request processed in 100us (14947us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320708.331599:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6114 00000100:00100000:2.0:1767320708.331616:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6115 00000100:00100000:2.0:1767320708.331620:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985494016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320708.331628:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320708.331651:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985494016:12345-0@lo:38:kworker.0 Request processed in 30us (14089us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320708.331657:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6115 00000100:00100000:2.0:1767320708.331722:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985493760 00000100:00100000:2.0:1767320708.331725:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985493760 00000100:00100000:2.0:1767320708.331751:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2019 00000100:00100000:2.0:1767320708.331755:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985493760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320708.331764:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320708.331795:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985493760:12345-0@lo:38:kworker.0 Request processed in 39us (14281us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320708.331800:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2019 00000100:00100000:3.0:1767320708.340493:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494272 00000100:00100000:3.0:1767320708.340512:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494272 00000100:00080000:1.0:1767320708.340541:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320708.340551:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320708.340552:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6116 00000100:00100000:1.0:1767320708.340557:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985493632:0@lo:38:kworker.0 00000100:00100000:3.0:1767320708.340560:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985494272:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767320708.340570:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320708.340574:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320708.340576:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985493760:0@lo:38:kworker.0 00010000:00020000:3.0:1767320708.340585:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320708.340586:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320708.340589:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320708.340592:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985494016:0@lo:38:kworker.0 00000100:00100000:3.0:1767320708.340667:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985494272:12345-0@lo:38:kworker.0 Request processed in 107us (11303us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320708.340675:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6116 00000100:00080000:1.0:1767320708.341062:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320708.341070:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320708.341073:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985494272:0@lo:38:kworker.0 00010000:00010000:1.0:1767320708.955513:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320709.980040:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320711.003777:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320712.027527:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320712.363110:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320712.363248:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920632320 00000100:00100000:2.0:1767320712.363260:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920632320 00000100:00100000:2.0:1767320712.363319:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6117 00000100:00100000:2.0:1767320712.363329:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920632320:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320712.363353:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320712.363420:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920632320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 90us (317us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320712.363427:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6117 00000100:00100000:3.0:1767320712.363674:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320712.363783:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920632448 00000100:00100000:0.0:1767320712.363789:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920632448 00000100:00100000:0.0:1767320712.363851:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5384 00000100:00100000:0.0:1767320712.363859:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920632448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320712.363891:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920632448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 34us (219us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320712.363896:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5384 00000100:00080000:1.0:1767320712.363956:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320712.923725:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320712.923749:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985494400:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320712.923818:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320712.924039:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494400 00000100:00100000:1.0:1767320712.924047:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494400 00000100:00080000:3.0:1767320712.924116:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320712.926016:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985494528:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320712.926098:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320712.926491:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494528 00000100:00100000:0.0:1767320712.926502:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494528 00000100:00100000:1.0:1767320712.926700:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2702 00000100:00100000:1.0:1767320712.926714:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985494400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320712.926748:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320712.926805:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985494400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (2986us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320712.926813:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2702 00000100:00080000:2.0:1767320712.926859:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320712.926922:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985494400:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320712.933820:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2703 00000100:00100000:0.0:1767320712.933843:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985494528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320712.933877:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320712.938184:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985494528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4313us (12048us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320712.938210:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2703 00000100:00100000:3.0:1767320712.938400:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985494528:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320713.052350:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320713.442541:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320713.442552:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320713.442555:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320713.442558:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320713.442562:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320713.442571:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5956 00000100:00080000:3.0:1767320713.442575:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320713.442582:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320713.442625:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ed80 x1853163985494656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320713.442653:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320713.442657:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320713.442659:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320713.442662:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5956 00000100:00080000:3.0:1767320713.442665:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320713.442668:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320713.442684:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4000 x1853163985494784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320713.442730:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320713.442734:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320713.442736:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320713.442740:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5956 00000100:00080000:3.0:1767320713.442743:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320713.442745:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320713.442758:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985495040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320713.442788:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320713.442792:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320713.442794:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320713.442797:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5956 00000100:00080000:3.0:1767320713.442799:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320713.442802:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320713.442813:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985495296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320713.442846:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320713.442860:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985495168:0@lo:400:kworker.0 00000100:00100000:3.0:1767320713.442920:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320713.442994:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985495168 00000100:00100000:3.0:1767320713.443007:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985495168 00000100:00100000:3.0:1767320713.443062:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5385 00000100:00100000:3.0:1767320713.443071:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985495168:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320713.443118:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985495168:12345-0@lo:400:kworker.0 Request processed in 49us (197us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320713.443124:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5385 00000100:00080000:3.0:1767320713.443145:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320713.443205:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985495168:0@lo:400:kworker.0 00000100:00100000:3.0:1767320713.443226:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320713.443233:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985494912:0@lo:400:kworker.0 00000100:00100000:3.0:1767320713.443256:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320713.443279:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494912 00000100:00100000:3.0:1767320713.443283:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494912 00000100:00100000:3.0:1767320713.443308:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5386 00000100:00100000:3.0:1767320713.443314:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985494912:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320713.443346:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985494912:12345-0@lo:400:kworker.0 Request processed in 32us (89us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320713.443352:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5386 00000100:00080000:3.0:1767320713.443364:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320713.443379:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985494912:0@lo:400:kworker.0 00000100:00100000:0.0:1767320713.443929:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985494656:0@lo:38:kworker.0 00000100:00100000:0.0:1767320713.443966:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320713.444001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985494784:0@lo:38:kworker.0 00000100:00100000:0.0:1767320713.444025:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320713.444047:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985495040:0@lo:38:kworker.0 00000100:00100000:0.0:1767320713.444065:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320713.444079:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985495296:0@lo:38:kworker.0 00000100:00100000:0.0:1767320713.444102:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320713.444130:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494784 00000100:00100000:0.0:1767320713.444135:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494784 00000100:00100000:0.0:1767320713.444157:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2020 00000100:00100000:0.0:1767320713.444195:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985494784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320713.444214:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320713.444246:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985494784:12345-0@lo:38:kworker.0 Request processed in 81us (219us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320713.444252:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2020 00000100:00100000:0.0:1767320713.444274:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985494656 00000100:00100000:0.0:1767320713.444278:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985494656 00000100:00100000:0.0:1767320713.444293:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985495040 00000100:00100000:0.0:1767320713.444295:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985495040 00000100:00100000:0.0:1767320713.444307:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985495296 00000100:00100000:0.0:1767320713.444310:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985495296 00000100:00100000:0.0:1767320713.444322:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6118 00000100:00100000:0.0:1767320713.444326:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985494656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320713.444334:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320713.444357:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985494656:12345-0@lo:38:kworker.0 Request processed in 30us (390us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320713.444363:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6118 00000100:00100000:0.0:1767320713.444375:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6119 00000100:00100000:0.0:1767320713.444379:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985495040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320713.444387:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320713.444408:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985495040:12345-0@lo:38:kworker.0 Request processed in 28us (342us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320713.444413:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6119 00000100:00100000:0.0:1767320713.444423:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6120 00000100:00100000:0.0:1767320713.444426:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce187fea050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985495296:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320713.444434:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320713.444449:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce187fea050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985495296:12345-0@lo:38:kworker.0 Request processed in 22us (346us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320713.444454:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6120 00000100:00080000:0.0:1767320713.445472:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320713.445480:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320713.445485:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985494656:0@lo:38:kworker.0 00000100:00080000:0.0:1767320713.445494:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320713.445497:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320713.445499:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985494784:0@lo:38:kworker.0 00000100:00080000:0.0:1767320713.445507:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320713.445510:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320713.445513:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985495040:0@lo:38:kworker.0 00000100:00080000:0.0:1767320713.445521:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320713.445523:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320713.445526:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985495296:0@lo:38:kworker.0 00010000:00010000:1.0:1767320714.075534:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320715.100490:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320716.126365:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320716.460949:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320716.461035:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920632832 00000100:00100000:3.0:1767320716.461045:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920632832 00000100:00100000:3.0:1767320716.461140:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5387 00000100:00100000:3.0:1767320716.461150:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920632832:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767320716.461259:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320716.461340:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920632832:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 191us (400us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320716.461360:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5387 00000400:00000100:1.0:1767320716.462716:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920632576 offset 0 length 520: 4 00000100:00100000:1.0:1767320716.462805:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320716.463286:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920632704 00000100:00100000:3.0:1767320716.463297:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920632704 00000100:00100000:3.0:1767320716.463343:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6121 00000100:00100000:3.0:1767320716.463352:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920632704:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320716.463377:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320716.463456:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920632704:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 102us (648us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320716.463463:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6121 00010000:00010000:1.0:1767320717.147405:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320717.531337:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320692/real 1767320692] req@ffff8ce1b3ffea00 x1853163985490688/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320717 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320717.531372:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3ffea00 x1853163985490688/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320717 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320717.531388:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320717.531415:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320717.531420:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985490688:0@lo:250:kworker.0 00000100:00080000:3.0:1767320717.531567:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320717.531573:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320717.531576:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320717.531581:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5941 00000100:00080000:3.0:1767320717.531585:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320717.531591:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320717.531623:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985495424/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320717.531745:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985495424:0@lo:250:kworker.0 00000400:00000100:0.0:1767320717.531793:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985495424 offset 0 length 520: 4 00000100:00080000:3.0:1767320717.531850:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320717.531855:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320717.531857:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320717.531860:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5961 00000100:00080000:3.0:1767320717.531863:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320717.531867:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320717.531883:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985495552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320717.538190:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985495552:0@lo:38:kworker.0 00000100:00100000:0.0:1767320717.538257:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320717.538749:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985495552 00000100:00100000:1.0:1767320717.538757:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985495552 00000100:00100000:1.0:1767320717.538810:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6122 00000100:00100000:1.0:1767320717.538818:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985495552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320717.538839:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320717.538889:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985495552:12345-0@lo:38:kworker.0 Request processed in 72us (631us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320717.538895:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6122 00000100:00080000:3.0:1767320717.538955:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320717.538959:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320717.538962:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320717.538966:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5961 00000100:00080000:3.0:1767320717.538969:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320717.538973:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320717.538997:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb800 x1853163985495680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320717.539029:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320717.539031:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320717.539032:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320717.539034:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5961 00000100:00080000:3.0:1767320717.539036:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320717.539038:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320717.539045:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9c00 x1853163985495936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320717.539061:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320717.539063:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320717.539064:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320717.539067:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5961 00000100:00080000:3.0:1767320717.539068:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320717.539071:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320717.539080:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9880 x1853163985496192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320717.539126:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985496064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320717.539144:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985495808:0@lo:400:kworker.0 00000100:00100000:3.0:1767320717.539192:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320717.539216:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320717.539218:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320717.539225:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985496320:0@lo:400:kworker.0 00000100:00100000:3.0:1767320717.539256:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320717.539267:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320717.539276:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985496448:0@lo:400:kworker.0 00000100:00100000:3.0:1767320717.539301:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496064 00000100:00100000:3.0:1767320717.539307:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496064 00000100:00100000:3.0:1767320717.539340:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985495808 00000100:00100000:3.0:1767320717.539343:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985495808 00000100:00100000:3.0:1767320717.539361:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5388 00000100:00100000:3.0:1767320717.539367:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985496064:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320717.539400:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320717.539425:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985496064:12345-0@lo:400:kworker.0 Request processed in 57us (233us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320717.539430:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5388 00000100:00100000:2.0:1767320717.539438:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496320 00000100:00100000:2.0:1767320717.539443:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496320 00000100:00100000:3.0:1767320717.539444:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5389 00000100:00100000:3.0:1767320717.539451:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985495808:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320717.539476:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985495808:12345-0@lo:400:kworker.0 Request processed in 26us (262us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320717.539482:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496448 00000100:00100000:2.0:1767320717.539486:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496448 00000100:00100000:3.0:1767320717.539496:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5389 00000100:00100000:2.0:1767320717.539514:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3493 00000100:00100000:3.0:1767320717.539520:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3494 00000100:00100000:2.0:1767320717.539525:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985496320:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320717.539529:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985496448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320717.539558:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985496448:12345-0@lo:400:kworker.0 Request processed in 32us (160us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320717.539562:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3494 00000100:00100000:2.0:1767320717.539578:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985496320:12345-0@lo:400:kworker.0 Request processed in 55us (321us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320717.539585:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3493 00000100:00100000:3.0:1767320717.539596:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985496320:0@lo:400:kworker.0 00000100:00100000:3.0:1767320717.539605:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985496064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320717.539625:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985496448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320717.539638:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985495808:0@lo:400:kworker.0 00000100:00080000:0.0:1767320717.539954:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320717.539957:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320717.539959:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320717.539975:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985495680:0@lo:38:kworker.0 00000100:00100000:0.0:1767320717.539997:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320717.540019:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985495936:0@lo:38:kworker.0 00000100:00100000:0.0:1767320717.540047:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320717.540063:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985496192:0@lo:38:kworker.0 00000100:00100000:0.0:1767320717.540077:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320717.540114:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320717.540119:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320717.540123:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985495552:0@lo:38:kworker.0 00000100:00100000:2.0:1767320717.540145:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985495680 00000100:00100000:2.0:1767320717.540150:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985495680 00000100:00100000:0.0:1767320717.540191:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2021 00000100:00100000:0.0:1767320717.540197:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985495680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320717.540208:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320717.540240:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985495680:12345-0@lo:38:kworker.0 Request processed in 44us (242us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320717.540244:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2021 00000100:00100000:1.0:1767320717.540340:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985495936 00000100:00100000:1.0:1767320717.540343:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985495936 00000100:00100000:1.0:1767320717.540362:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496192 00000100:00100000:1.0:1767320717.540364:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496192 00000100:00100000:1.0:1767320717.540377:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6123 00000100:00100000:1.0:1767320717.540381:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985495936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320717.540391:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320717.540399:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320717.540406:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320717.540411:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985495680:0@lo:38:kworker.0 00000100:00100000:1.0:1767320717.540419:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985495936:12345-0@lo:38:kworker.0 Request processed in 35us (369us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320717.540425:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6123 00000100:00080000:2.0:1767320717.540425:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320717.540433:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320717.540437:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985495936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320717.540439:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6124 00000100:00100000:1.0:1767320717.540447:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985496192:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320717.540453:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320717.540475:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985496192:12345-0@lo:38:kworker.0 Request processed in 29us (398us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320717.540480:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6124 00000100:00080000:2.0:1767320717.540492:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320717.540498:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320717.540502:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985496192:0@lo:38:kworker.0 00000100:00100000:0.0:1767320718.044684:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985496576:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320718.044772:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320718.044905:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985496704:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320718.044920:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496576 00000100:00100000:1.0:1767320718.044931:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496576 00000100:00100000:3.0:1767320718.044937:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320718.044984:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496704 00000100:00100000:1.0:1767320718.044988:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496704 00000100:00100000:1.0:1767320718.045013:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2704 00000100:00100000:3.0:1767320718.045021:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2705 00000100:00100000:1.0:1767320718.045025:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985496576:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320718.045036:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985496704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320718.045061:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:1.0:1767320718.045067:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320718.047277:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985496704:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320718.049471:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985496576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4442us (4691us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320718.049486:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2704 00000100:00100000:0.0:1767320718.050837:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985496576:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320718.050858:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985496704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5823us (5917us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320718.050869:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2705 00010000:00010000:2.0:1767320718.171365:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320719.197018:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320720.220751:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320721.243375:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320721.578308:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320721.578363:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320721.579135:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920633088 00000100:00100000:3.0:1767320721.579136:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920632960 00000100:00100000:3.0:1767320721.579148:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920632960 00000100:00100000:0.0:1767320721.579149:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920633088 00000100:00100000:3.0:1767320721.579241:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6125 00000100:00100000:3.0:1767320721.579249:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920632960:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320721.579273:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320721.579352:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920632960:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 101us (987us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320721.579361:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6125 00000100:00100000:3.0:1767320721.579381:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5390 00000100:00100000:3.0:1767320721.579390:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920633088:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767320721.579396:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320721.579431:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920633088:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 43us (1133us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320721.579438:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5390 00010000:00010000:1.0:1767320722.267872:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320722.651763:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320722.651777:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320722.651780:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320722.651784:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320722.651789:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320722.651797:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5965 00000100:00080000:3.0:1767320722.651803:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320722.651809:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320722.651864:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfa680 x1853163985496832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320722.651895:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320722.651899:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320722.651901:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320722.651905:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5965 00000100:00080000:3.0:1767320722.651907:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320722.651911:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320722.651925:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985496960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320722.652023:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985497088:0@lo:400:kworker.0 00000100:00100000:1.0:1767320722.652093:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320722.652208:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985496832:0@lo:38:kworker.0 00000100:00100000:2.0:1767320722.652237:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320722.652262:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985496960:0@lo:38:kworker.0 00000100:00100000:2.0:1767320722.652284:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320722.652334:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497088 00000100:00100000:2.0:1767320722.652342:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497088 00000100:00100000:2.0:1767320722.652389:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496832 00000100:00100000:2.0:1767320722.652392:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496832 00000100:00100000:2.0:1767320722.652414:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5391 00000100:00080000:0.0:1767320722.652415:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320722.652426:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985497088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.652433:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985496960 00000100:00100000:0.0:1767320722.652438:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985496960 00000100:00080000:3.0:1767320722.652510:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320722.652513:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320722.652515:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320722.652519:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5965 00000100:00080000:3.0:1767320722.652521:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320722.652525:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320722.652541:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9c00 x1853163985497216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320722.652572:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320722.652575:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320722.652576:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320722.652580:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5965 00000100:00080000:3.0:1767320722.652582:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320722.652585:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320722.652595:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985497472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320722.652622:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2022 00000100:00100000:0.0:1767320722.652630:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985496960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320722.652648:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320722.652683:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985496960:12345-0@lo:38:kworker.0 Request processed in 53us (397us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320722.652690:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2022 00000100:00100000:0.0:1767320722.652719:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985497728:0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.652746:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320722.652781:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497728 00000100:00100000:0.0:1767320722.652785:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497728 00000100:00100000:0.0:1767320722.652818:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3495 00000100:00100000:0.0:1767320722.652825:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985497728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.652859:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985497728:12345-0@lo:400:kworker.0 Request processed in 35us (112us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320722.652865:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3495 00000100:00080000:0.0:1767320722.652879:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320722.652900:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985497728:0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.652924:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6126 00000100:00100000:0.0:1767320722.652929:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985496832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320722.652940:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320722.652964:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985496832:12345-0@lo:38:kworker.0 Request processed in 34us (726us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320722.652971:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6126 00000100:00100000:0.0:1767320722.652995:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985497600:0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.653033:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320722.653054:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320722.653060:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985497344:0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.653079:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320722.653100:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497600 00000100:00100000:0.0:1767320722.653104:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497600 00000100:00100000:0.0:1767320722.653129:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3496 00000100:00100000:0.0:1767320722.653137:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985497600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.653209:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985497600:12345-0@lo:400:kworker.0 Request processed in 74us (175us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320722.653215:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3496 00000100:00080000:0.0:1767320722.653228:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320722.653258:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497344 00000100:00100000:0.0:1767320722.653262:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497344 00000100:00100000:0.0:1767320722.653288:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5392 00000100:00100000:0.0:1767320722.653294:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985497344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.653321:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985497344:12345-0@lo:400:kworker.0 Request processed in 28us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320722.653327:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5392 00000100:00080000:0.0:1767320722.653340:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320722.653357:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985497344:0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.653366:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985497600:0@lo:400:kworker.0 00000100:00100000:0.0:1767320722.653415:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985497216:0@lo:38:kworker.0 00000100:00100000:0.0:1767320722.653437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320722.653452:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985497472:0@lo:38:kworker.0 00000100:00100000:0.0:1767320722.653493:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320722.653515:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320722.653520:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320722.653523:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985496832:0@lo:38:kworker.0 00000100:00080000:0.0:1767320722.653530:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320722.653533:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320722.653536:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985496960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320722.653559:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497216 00000100:00100000:0.0:1767320722.653562:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497216 00000100:00100000:0.0:1767320722.653579:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497472 00000100:00100000:0.0:1767320722.653582:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497472 00000100:00100000:0.0:1767320722.653594:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6127 00000100:00100000:0.0:1767320722.653597:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985497216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320722.653607:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320722.653634:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985497216:12345-0@lo:38:kworker.0 Request processed in 35us (196us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320722.653640:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6127 00000100:00100000:0.0:1767320722.653652:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6128 00000100:00100000:0.0:1767320722.653656:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985497472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320722.653665:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320722.653687:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985497472:12345-0@lo:38:kworker.0 Request processed in 30us (193us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320722.653693:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6128 00000100:00080000:0.0:1767320722.653721:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320722.653725:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320722.653727:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985497216:0@lo:38:kworker.0 00000100:00080000:0.0:1767320722.653733:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320722.653736:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320722.653739:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985497472:0@lo:38:kworker.0 00000100:00100000:2.0:1767320722.657081:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985497088:12345-0@lo:400:kworker.0 Request processed in 4647us (4975us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320722.657095:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5391 00000100:00100000:1.0:1767320722.660935:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985497088:0@lo:400:kworker.0 00000100:00100000:0.0:1767320723.165241:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320723.165266:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985497856:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320723.165342:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320723.165699:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497856 00000100:00100000:2.0:1767320723.165708:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497856 00000100:00100000:2.0:1767320723.165760:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2706 00000100:00100000:2.0:1767320723.165768:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985497856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320723.165797:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320723.165849:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985497856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (505us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320723.165856:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2706 00000100:00100000:0.0:1767320723.166132:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985497984:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320723.166159:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320723.166238:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985497856:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320723.166476:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985497984 00000100:00100000:3.0:1767320723.166482:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985497984 00000100:00100000:3.0:1767320723.166513:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2707 00000100:00100000:3.0:1767320723.166521:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985497984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320723.166536:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320723.166576:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985497984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (416us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320723.166582:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2707 00000100:00100000:0.0:1767320723.166696:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985497984:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320723.291452:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320724.315715:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320725.339485:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320726.363428:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320726.698836:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320726.698949:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320726.701717:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920633344 00000100:00100000:3.0:1767320726.701732:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920633344 00000100:00100000:3.0:1767320726.701801:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920633216 00000100:00100000:3.0:1767320726.701803:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920633216 00000100:00100000:3.0:1767320726.701825:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5393 00000100:00100000:3.0:1767320726.701832:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920633344:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767320726.701974:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320726.702010:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6129 00000100:00100000:1.0:1767320726.702016:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920633216:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320726.702036:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320726.702220:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920633344:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 386us (3394us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320726.702235:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5393 00000100:00100000:1.0:1767320726.702659:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920633216:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 641us (3708us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320726.702667:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6129 00010000:00010000:1.0:1767320727.389848:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320727.771258:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320727.771268:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320727.771271:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320727.771275:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320727.771280:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320727.771288:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5970 00000100:00080000:3.0:1767320727.771292:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320727.771300:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320727.771343:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985498112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320727.771370:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320727.771374:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320727.771376:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320727.771379:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5970 00000100:00080000:3.0:1767320727.771381:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320727.771385:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320727.771397:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9c00 x1853163985498240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320727.771444:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320727.771448:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320727.771450:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320727.771454:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5970 00000100:00080000:3.0:1767320727.771457:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320727.771461:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320727.771473:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfa680 x1853163985498496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320727.771513:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320727.771516:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320727.771518:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320727.771521:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5970 00000100:00080000:3.0:1767320727.771523:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320727.771526:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320727.771536:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb800 x1853163985498752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320727.771575:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985498880:0@lo:400:kworker.0 00000100:00100000:3.0:1767320727.771642:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320727.771704:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320727.771710:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985498624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320727.771732:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320727.771898:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985499008:0@lo:400:kworker.0 00000100:00100000:2.0:1767320727.771956:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320727.771986:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320727.771992:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985498368:0@lo:400:kworker.0 00000100:00100000:2.0:1767320727.772014:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320727.772228:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985498880 00000100:00100000:0.0:1767320727.772237:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985498880 00000100:00100000:0.0:1767320727.772286:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499008 00000100:00100000:0.0:1767320727.772289:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499008 00000100:00100000:0.0:1767320727.772311:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3497 00000100:00100000:0.0:1767320727.772318:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985498880:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320727.772366:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985498880:12345-0@lo:400:kworker.0 Request processed in 48us (724us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320727.772373:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3497 00000100:00100000:0.0:1767320727.772389:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3498 00000100:00100000:0.0:1767320727.772395:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985499008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320727.772427:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985499008:12345-0@lo:400:kworker.0 Request processed in 32us (470us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320727.772432:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3498 00000100:00100000:0.0:1767320727.772610:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985498112:0@lo:38:kworker.0 00000100:00100000:0.0:1767320727.772642:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320727.772665:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985498240:0@lo:38:kworker.0 00000100:00100000:0.0:1767320727.772689:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320727.772711:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985498496:0@lo:38:kworker.0 00000100:00100000:0.0:1767320727.772731:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320727.772746:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985498752:0@lo:38:kworker.0 00000100:00100000:0.0:1767320727.772765:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320727.772791:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985498240 00000100:00100000:0.0:1767320727.772795:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985498240 00000100:00100000:0.0:1767320727.772813:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2023 00000100:00100000:0.0:1767320727.772817:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985498240:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320727.772835:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320727.772864:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985498240:12345-0@lo:38:kworker.0 Request processed in 45us (174us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320727.772882:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2023 00000100:00100000:0.0:1767320727.772899:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985498624 00000100:00100000:0.0:1767320727.772903:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985498624 00000100:00100000:0.0:1767320727.772937:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985498368 00000100:00100000:0.0:1767320727.772940:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985498368 00000100:00100000:0.0:1767320727.772955:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985498112 00000100:00100000:0.0:1767320727.772957:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985498112 00000100:00100000:0.0:1767320727.772963:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985498496 00000100:00100000:0.0:1767320727.772965:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985498496 00000100:00100000:0.0:1767320727.772972:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985498752 00000100:00100000:0.0:1767320727.772974:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985498752 00000100:00100000:0.0:1767320727.772982:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5394 00000100:00100000:0.0:1767320727.772987:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985498624:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320727.773016:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985498624:12345-0@lo:400:kworker.0 Request processed in 28us (1282us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320727.773022:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5394 00000100:00100000:0.0:1767320727.773033:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5395 00000100:00100000:0.0:1767320727.773039:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985498368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320727.773061:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985498368:12345-0@lo:400:kworker.0 Request processed in 23us (1046us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320727.773066:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5395 00000100:00100000:0.0:1767320727.773076:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6130 00000100:00100000:0.0:1767320727.773079:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985498112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320727.773088:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320727.773107:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985498112:12345-0@lo:38:kworker.0 Request processed in 26us (464us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320727.773112:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6130 00000100:00100000:0.0:1767320727.773121:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6131 00000100:00100000:0.0:1767320727.773124:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985498496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320727.773133:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320727.773152:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985498496:12345-0@lo:38:kworker.0 Request processed in 26us (420us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320727.773157:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6131 00000100:00100000:0.0:1767320727.773192:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6132 00000100:00100000:0.0:1767320727.773196:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985498752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320727.773205:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320727.773226:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985498752:12345-0@lo:38:kworker.0 Request processed in 28us (460us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320727.773231:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6132 00000100:00100000:0.0:1767320727.773314:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985498368:0@lo:400:kworker.0 00000100:00100000:0.0:1767320727.773324:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985499008:0@lo:400:kworker.0 00000100:00080000:0.0:1767320727.773352:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320727.773356:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320727.773359:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985498112:0@lo:38:kworker.0 00000100:00080000:0.0:1767320727.773367:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320727.773370:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320727.773372:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985498240:0@lo:38:kworker.0 00000100:00080000:0.0:1767320727.773379:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320727.773382:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320727.773384:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985498496:0@lo:38:kworker.0 00000100:00080000:0.0:1767320727.773391:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320727.773393:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320727.773396:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985498752:0@lo:38:kworker.0 00000100:00100000:3.0:1767320727.773800:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985498624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320727.773811:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985498880:0@lo:400:kworker.0 00000100:00080000:2.0:1767320727.774144:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320727.774151:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320727.774153:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320728.283487:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320728.283525:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985499136:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320728.283597:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320728.283785:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499136 00000100:00100000:2.0:1767320728.283795:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499136 00000100:00100000:2.0:1767320728.283864:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2708 00000100:00100000:2.0:1767320728.283874:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985499136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320728.283903:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320728.283980:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985499136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320728.283989:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2708 00000100:00100000:0.0:1767320728.284085:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985499136:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320728.284369:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985499264:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320728.284397:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320728.285035:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499264 00000100:00100000:3.0:1767320728.285042:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499264 00000100:00100000:3.0:1767320728.285087:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2709 00000100:00100000:3.0:1767320728.285095:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985499264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320728.285116:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320728.285208:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985499264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 114us (809us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320728.285219:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2709 00000100:00100000:0.0:1767320728.285392:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985499264:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320728.414754:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320729.435687:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320730.459557:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320731.483522:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320731.818949:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320731.819010:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320731.823411:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920633600 00000100:00100000:1.0:1767320731.823428:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920633600 00000100:00100000:1.0:1767320731.823510:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920633472 00000100:00100000:1.0:1767320731.823514:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920633472 00000100:00100000:1.0:1767320731.823550:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5396 00000100:00100000:1.0:1767320731.823560:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920633600:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320731.823662:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920633600:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 103us (4722us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320731.823669:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5396 00000100:00080000:2.0:1767320731.823800:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320731.823861:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6133 00000100:00100000:2.0:1767320731.823868:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920633472:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320731.823890:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320731.823919:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920633472:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 50us (4908us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320731.823925:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6133 00010000:00010000:1.0:1767320732.508263:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320732.891629:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320732.891642:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320732.891645:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320732.891648:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320732.891654:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320732.891663:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5976 00000100:00080000:2.0:1767320732.891667:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320732.891683:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320732.891736:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89f80 x1853163985499392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320732.891768:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320732.891772:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320732.891774:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320732.891778:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5976 00000100:00080000:2.0:1767320732.891781:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320732.891785:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320732.891802:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8bb80 x1853163985499520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320732.891858:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320732.891863:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320732.891865:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320732.891871:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5976 00000100:00080000:2.0:1767320732.891874:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320732.891878:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320732.891893:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985499392:0@lo:38:kworker.0 00000100:00100000:2.0:1767320732.891894:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b800 x1853163985499776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320732.891954:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320732.891957:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320732.891959:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320732.891962:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5976 00000100:00080000:2.0:1767320732.891965:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320732.891969:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320732.891982:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07800 x1853163985500032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320732.892009:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320732.892047:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985499904:0@lo:400:kworker.0 00000100:00100000:0.0:1767320732.892105:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985499520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320732.892109:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320732.892153:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320732.892189:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320732.892218:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985499776:0@lo:38:kworker.0 00000100:00100000:0.0:1767320732.892233:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320732.892244:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985500032:0@lo:38:kworker.0 00000100:00100000:0.0:1767320732.892255:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320732.892282:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499520 00000100:00100000:0.0:1767320732.892290:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499520 00000100:00100000:3.0:1767320732.892306:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985499648:0@lo:400:kworker.0 00000100:00100000:3.0:1767320732.892339:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320732.892359:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320732.892362:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985500288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320732.892380:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320732.892412:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499392 00000100:00100000:3.0:1767320732.892419:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499392 00000100:00100000:3.0:1767320732.892457:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499904 00000100:00100000:3.0:1767320732.892460:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499904 00000100:00100000:3.0:1767320732.892499:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499776 00000100:00100000:3.0:1767320732.892502:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499776 00000100:00100000:0.0:1767320732.892513:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2024 00000100:00100000:3.0:1767320732.892515:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500032 00000100:00100000:0.0:1767320732.892520:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985499520:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320732.892531:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500032 00010000:00020000:0.0:1767320732.892541:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320732.892546:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985499648 00000100:00100000:3.0:1767320732.892552:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985499648 00000100:00100000:3.0:1767320732.892584:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5397 00000100:00100000:0.0:1767320732.892593:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985499520:12345-0@lo:38:kworker.0 Request processed in 71us (402us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320732.892599:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985499904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320732.892610:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2024 00000100:00100000:0.0:1767320732.892643:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5398 00000100:00100000:0.0:1767320732.892653:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985499648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320732.892693:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985499904:12345-0@lo:400:kworker.0 Request processed in 97us (584us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320732.892703:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985499648:12345-0@lo:400:kworker.0 Request processed in 51us (360us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320732.892711:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5397 00000100:00100000:0.0:1767320732.892721:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5398 00000100:00100000:3.0:1767320732.892759:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6134 00000100:00100000:3.0:1767320732.892767:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985499392:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320732.892784:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320732.892822:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985499392:12345-0@lo:38:kworker.0 Request processed in 54us (820us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320732.892875:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985500160:0@lo:400:kworker.0 00000100:00100000:2.0:1767320732.892920:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320732.892925:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6135 00000100:00100000:0.0:1767320732.892934:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985499776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320732.892948:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320732.892983:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985499904:0@lo:400:kworker.0 00000100:00100000:0.0:1767320732.892993:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985499776:12345-0@lo:38:kworker.0 Request processed in 58us (756us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320732.893005:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6135 00000100:00080000:2.0:1767320732.893030:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320732.893031:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6136 00000100:00100000:0.0:1767320732.893046:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985500032:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320732.893050:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500288 00000100:00100000:2.0:1767320732.893056:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500288 00010000:00020000:0.0:1767320732.893059:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320732.893101:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500160 00000100:00100000:0.0:1767320732.893102:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985500032:12345-0@lo:38:kworker.0 Request processed in 61us (844us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320732.893105:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500160 00000100:00100000:0.0:1767320732.893118:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6136 00000100:00100000:2.0:1767320732.893137:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3499 00000100:00100000:2.0:1767320732.893144:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985500288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320732.893210:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985499648:0@lo:400:kworker.0 00000100:00080000:0.0:1767320732.893234:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320732.893238:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320732.893240:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985499776:0@lo:38:kworker.0 00000100:00080000:0.0:1767320732.893244:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320732.893246:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320732.893248:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985500032:0@lo:38:kworker.0 00000100:00080000:0.0:1767320732.893254:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320732.893257:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320732.893258:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985499392:0@lo:38:kworker.0 00000100:00080000:0.0:1767320732.893263:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320732.893265:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320732.893267:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985499520:0@lo:38:kworker.0 00000100:00100000:0.0:1767320732.893335:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3500 00000100:00100000:0.0:1767320732.893344:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985500160:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320732.893392:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985500160:12345-0@lo:400:kworker.0 Request processed in 49us (478us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320732.893398:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3500 00000100:00100000:2.0:1767320732.893451:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985500288:12345-0@lo:400:kworker.0 Request processed in 304us (1065us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320732.893461:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3499 00000100:00100000:0.0:1767320732.893481:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985500288:0@lo:400:kworker.0 00000100:00080000:2.0:1767320732.893492:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320732.893499:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320732.893534:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985500160:0@lo:400:kworker.0 00000100:00100000:3.0:1767320732.894504:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6134 00000100:00100000:0.0:1767320733.403872:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985500416:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320733.403998:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320733.404013:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320733.404095:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500416 00000100:00100000:1.0:1767320733.404099:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985500544:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320733.404107:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500416 00000100:00100000:2.0:1767320733.404227:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2710 00000100:00100000:2.0:1767320733.404239:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985500416:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320733.404277:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320733.404318:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320733.404342:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985500416:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (343us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320733.404351:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2710 00000100:00100000:0.0:1767320733.404493:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985500416:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320733.404756:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500544 00000100:00100000:2.0:1767320733.404761:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500544 00000100:00100000:3.0:1767320733.405214:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2711 00000100:00100000:3.0:1767320733.405229:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985500544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320733.405555:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320733.405641:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985500544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 413us (1320us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320733.405649:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2711 00000100:00100000:1.0:1767320733.405877:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985500544:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320733.531491:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320734.555427:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320735.580301:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320736.606434:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320736.939884:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320736.940347:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920633856 00000100:00100000:1.0:1767320736.940362:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920633856 00000100:00100000:1.0:1767320736.940474:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5399 00000100:00100000:1.0:1767320736.940486:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920633856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320736.940602:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920633856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 118us (734us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320736.940611:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5399 00000100:00080000:3.0:1767320736.940687:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320736.940990:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320736.941202:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920633728 00000100:00100000:1.0:1767320736.941213:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920633728 00000100:00100000:1.0:1767320736.941520:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6137 00000100:00100000:1.0:1767320736.941529:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920633728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320736.941552:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320736.941633:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920633728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 103us (653us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320736.941641:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6137 00010000:00010000:0.0:1767320737.632323:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320738.011362:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320738.011377:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320738.011382:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320738.011387:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320738.011393:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320738.011404:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5981 00000100:00080000:3.0:1767320738.011410:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320738.011419:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320738.011473:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4380 x1853163985500672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320738.011548:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985500672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320738.011650:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320738.011879:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500672 00000100:00100000:1.0:1767320738.011889:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500672 00000100:00100000:1.0:1767320738.011980:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6138 00000100:00100000:1.0:1767320738.011988:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985500672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320738.012015:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320738.012069:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320738.012073:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320738.012075:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320738.012079:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5981 00000100:00080000:3.0:1767320738.012082:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320738.012086:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320738.012105:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6680 x1853163985500800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320738.012149:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985500800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320738.012214:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320738.020210:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985500672:12345-0@lo:38:kworker.0 Request processed in 8205us (8543us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320738.020229:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6138 00000100:00100000:0.0:1767320738.020255:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500800 00000100:00100000:0.0:1767320738.020269:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500800 00000100:00100000:0.0:1767320738.020328:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2025 00000100:00100000:0.0:1767320738.020337:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985500800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320738.020375:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320738.020455:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320738.020463:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320738.020464:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985500800:12345-0@lo:38:kworker.0 Request processed in 127us (8249us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320738.020469:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985500672:0@lo:38:kworker.0 00000100:00100000:0.0:1767320738.020471:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2025 00000100:00080000:2.0:1767320738.020509:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320738.020513:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320738.020517:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985500800:0@lo:38:kworker.0 00000100:00080000:3.0:1767320738.020810:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320738.020816:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320738.020820:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320738.020825:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5981 00000100:00080000:3.0:1767320738.020830:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320738.020836:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320738.020861:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7b80 x1853163985501056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320738.020892:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320738.020895:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320738.020897:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320738.020900:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5981 00000100:00080000:3.0:1767320738.020903:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320738.020907:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320738.020919:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5500 x1853163985501312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320738.020974:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985500928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320738.021037:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320738.021072:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320738.021078:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985501568:0@lo:400:kworker.0 00000100:00100000:3.0:1767320738.021105:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320738.021149:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985501184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320738.021194:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320738.021214:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320738.021219:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985501440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320738.021249:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320738.021276:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985500928 00000100:00100000:3.0:1767320738.021281:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985500928 00000100:00100000:3.0:1767320738.021332:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985501184 00000100:00100000:3.0:1767320738.021335:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985501184 00000100:00100000:3.0:1767320738.021355:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5400 00000100:00100000:3.0:1767320738.021362:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985500928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320738.021408:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985500928:12345-0@lo:400:kworker.0 Request processed in 47us (371us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320738.021415:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5400 00000100:00100000:3.0:1767320738.021429:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5401 00000100:00100000:3.0:1767320738.021435:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985501184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320738.021463:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985501184:12345-0@lo:400:kworker.0 Request processed in 29us (269us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320738.021469:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5401 00000100:00080000:3.0:1767320738.021485:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320738.021513:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985501184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320738.021538:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985500928:0@lo:400:kworker.0 00000100:00100000:2.0:1767320738.021626:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985501568 00000100:00100000:2.0:1767320738.021631:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985501568 00000100:00100000:2.0:1767320738.021676:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985501440 00000100:00100000:2.0:1767320738.021679:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985501440 00000100:00100000:2.0:1767320738.021700:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3501 00000100:00100000:2.0:1767320738.021706:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985501568:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320738.021761:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985501568:12345-0@lo:400:kworker.0 Request processed in 54us (654us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320738.021766:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3501 00000100:00100000:2.0:1767320738.021781:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3502 00000100:00100000:2.0:1767320738.021787:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a03450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985501440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320738.021818:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a03450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985501440:12345-0@lo:400:kworker.0 Request processed in 32us (568us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320738.021823:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3502 00000100:00100000:2.0:1767320738.022457:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985501440:0@lo:400:kworker.0 00000100:00100000:2.0:1767320738.022492:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985501568:0@lo:400:kworker.0 00000100:00100000:2.0:1767320738.022517:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985501056:0@lo:38:kworker.0 00000100:00100000:2.0:1767320738.022554:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320738.022582:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985501312:0@lo:38:kworker.0 00000100:00100000:2.0:1767320738.022604:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320738.022636:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985501056 00000100:00100000:2.0:1767320738.022641:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985501056 00000100:00100000:2.0:1767320738.022662:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985501312 00000100:00100000:2.0:1767320738.022665:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985501312 00000100:00100000:2.0:1767320738.022675:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6139 00000100:00100000:2.0:1767320738.022681:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985501056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320738.022699:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320738.022746:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985501056:12345-0@lo:38:kworker.0 Request processed in 63us (191us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320738.022754:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6139 00000100:00100000:2.0:1767320738.022768:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6140 00000100:00100000:2.0:1767320738.022771:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985501312:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320738.022780:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320738.022802:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985501312:12345-0@lo:38:kworker.0 Request processed in 29us (197us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320738.022807:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6140 00000100:00080000:2.0:1767320738.022832:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320738.022837:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320738.022840:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985501056:0@lo:38:kworker.0 00000100:00080000:2.0:1767320738.022848:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320738.022851:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320738.022853:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985501312:0@lo:38:kworker.0 00000100:00080000:3.0:1767320738.023385:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320738.023392:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320738.523673:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320738.523707:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985501696:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320738.523811:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320738.524010:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985501824:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320738.524037:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320738.524286:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985501696 00000100:00100000:3.0:1767320738.524295:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985501696 00000100:00100000:3.0:1767320738.524345:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985501824 00000100:00100000:3.0:1767320738.524347:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985501824 00000100:00100000:3.0:1767320738.524367:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2712 00000100:00100000:3.0:1767320738.524375:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985501696:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320738.524411:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320738.524455:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985501696:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (646us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320738.524461:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2712 00000100:00100000:3.0:1767320738.524476:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2713 00000100:00100000:3.0:1767320738.524480:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985501824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320738.524498:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320738.524528:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985501824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 46us (488us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320738.524534:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2713 00000100:00100000:0.0:1767320738.524633:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985501824:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320738.524741:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985501696:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320738.651850:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320739.678087:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320740.699448:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320741.723411:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320742.062462:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767320742.062545:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920633984 offset 0 length 520: 4 00000100:00100000:2.0:1767320742.062562:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320742.067933:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920634240 00000100:00100000:0.0:1767320742.067946:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920634240 00000100:00100000:0.0:1767320742.068021:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920634112 00000100:00100000:0.0:1767320742.068023:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920634112 00000100:00100000:0.0:1767320742.068049:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5402 00000100:00100000:0.0:1767320742.068056:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920634240:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320742.068125:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920634240:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 70us (5673us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320742.068133:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5402 00000100:00100000:0.0:1767320742.068147:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6141 00000100:00100000:0.0:1767320742.068150:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920634112:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320742.068180:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320742.068196:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920634112:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (5633us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320742.068200:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6141 00000100:00080000:3.0:1767320742.068365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320742.107282:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320717/real 1767320717] req@ffff8ce1a4df8000 x1853163985495424/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320742 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320742.107309:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a4df8000 x1853163985495424/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320742 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320742.107327:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320742.107350:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320742.107354:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985495424:0@lo:250:kworker.0 00000100:00080000:0.0:1767320742.107499:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320742.107503:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320742.107505:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320742.107509:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5965 00000100:00080000:0.0:1767320742.107512:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320742.107517:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320742.107545:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612e680 x1853163985501952/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320742.107587:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985501952:0@lo:250:kworker.0 00000400:00000100:2.0:1767320742.107633:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985501952 offset 0 length 520: 4 00000100:00080000:0.0:1767320742.107714:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320742.107718:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320742.107720:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320742.107724:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5986 00000100:00080000:0.0:1767320742.107727:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320742.107730:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320742.107747:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c380 x1853163985502080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320742.110914:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985502080:0@lo:38:kworker.0 00000100:00100000:2.0:1767320742.110977:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320742.111158:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320742.111171:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320742.111174:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320742.111177:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5986 00000100:00080000:0.0:1767320742.111180:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320742.111184:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320742.111209:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985502208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320742.111236:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320742.111238:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320742.111239:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320742.111241:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5986 00000100:00080000:0.0:1767320742.111243:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320742.111245:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320742.111254:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320742.111260:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985502336:0@lo:400:kworker.0 00000100:00100000:0.0:1767320742.111282:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320742.111315:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985502208:0@lo:38:kworker.0 00000100:00100000:2.0:1767320742.111340:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320742.111422:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502208 00000100:00100000:3.0:1767320742.111431:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502208 00000100:00100000:3.0:1767320742.111564:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2026 00000100:00100000:3.0:1767320742.111572:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985502208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320742.111592:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320742.111637:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985502208:12345-0@lo:38:kworker.0 Request processed in 64us (295us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320742.111644:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2026 00000100:00080000:2.0:1767320742.111794:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320742.111799:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320742.111804:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985502208:0@lo:38:kworker.0 00000100:00100000:0.0:1767320742.115097:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985502464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320742.115147:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320742.115150:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320742.115153:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320742.115156:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5986 00000100:00080000:0.0:1767320742.115158:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320742.115174:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320742.115179:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320742.115191:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985502592:0@lo:400:kworker.0 00000100:00100000:0.0:1767320742.115233:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320742.115273:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985502464:0@lo:38:kworker.0 00000100:00100000:1.0:1767320742.116250:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502080 00000100:00100000:1.0:1767320742.116259:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502080 00000100:00100000:1.0:1767320742.116305:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502336 00000100:00100000:1.0:1767320742.116307:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502336 00000100:00100000:1.0:1767320742.116349:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5403 00000100:00100000:1.0:1767320742.116358:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985502336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320742.116427:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985502336:12345-0@lo:400:kworker.0 Request processed in 71us (5143us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320742.116433:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5403 00000100:00100000:1.0:1767320742.116446:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6142 00000100:00100000:1.0:1767320742.116449:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985502080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320742.116465:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320742.116487:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985502080:12345-0@lo:38:kworker.0 Request processed in 37us (5509us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320742.116491:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6142 00000100:00080000:1.0:1767320742.116509:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320742.116534:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985502336:0@lo:400:kworker.0 00000100:00100000:1.0:1767320742.116580:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552d500 x1853163985502720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320742.116603:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320742.116613:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985502848:0@lo:400:kworker.0 00000100:00100000:1.0:1767320742.116642:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320742.116685:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502848 00000100:00100000:1.0:1767320742.116689:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502848 00000100:00100000:1.0:1767320742.116724:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3503 00000100:00100000:1.0:1767320742.116729:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985502848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320742.116763:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985502848:12345-0@lo:400:kworker.0 Request processed in 33us (119us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320742.116768:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3503 00000100:00080000:1.0:1767320742.116779:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320742.116793:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985502848:0@lo:400:kworker.0 00000100:00100000:1.0:1767320742.116824:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985502976:0@lo:400:kworker.0 00000100:00100000:1.0:1767320742.116847:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320742.116879:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502976 00000100:00100000:1.0:1767320742.116882:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502976 00000100:00100000:1.0:1767320742.116907:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3504 00000100:00100000:1.0:1767320742.116911:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985502976:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320742.116920:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320742.116938:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985502976:12345-0@lo:400:kworker.0 Request processed in 26us (90us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320742.116942:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3504 00000100:00100000:1.0:1767320742.116959:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985502976:0@lo:400:kworker.0 00000100:00100000:1.0:1767320742.121196:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320742.121390:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320742.121397:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320742.121401:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985502080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320742.121426:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985502720:0@lo:38:kworker.0 00000100:00100000:1.0:1767320742.121466:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320742.125208:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502592 00000100:00100000:3.0:1767320742.125222:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502592 00000100:00080000:3.0:1767320742.125252:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320742.125295:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502464 00000100:00100000:3.0:1767320742.125297:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502464 00000100:00100000:3.0:1767320742.125322:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5404 00000100:00100000:3.0:1767320742.125333:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985502592:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320742.125372:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6143 00000100:00100000:2.0:1767320742.125380:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985502464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320742.125401:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320742.125421:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985502592:12345-0@lo:400:kworker.0 Request processed in 91us (10185us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320742.125427:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5404 00000100:00100000:2.0:1767320742.125445:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985502464:12345-0@lo:38:kworker.0 Request processed in 64us (4252us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320742.125453:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6143 00000100:00100000:3.0:1767320742.125472:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985502592:0@lo:400:kworker.0 00000100:00080000:1.0:1767320742.125578:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320742.125583:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320742.125585:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985502464:0@lo:38:kworker.0 00000100:00100000:2.0:1767320742.125670:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985502720 00000100:00100000:2.0:1767320742.125675:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985502720 00000100:00100000:2.0:1767320742.125755:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6144 00000100:00100000:2.0:1767320742.125760:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985502720:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320742.125771:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320742.125812:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985502720:12345-0@lo:38:kworker.0 Request processed in 50us (4344us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320742.125819:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6144 00000100:00080000:1.0:1767320742.125872:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320742.125877:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320742.125880:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985502720:0@lo:38:kworker.0 00010000:00010000:2.0:1767320742.747351:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320743.643599:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985503104:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320743.643685:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320743.643916:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985503104 00000100:00100000:2.0:1767320743.643926:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985503104 00000100:00100000:2.0:1767320743.643979:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2714 00000100:00100000:2.0:1767320743.643989:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985503104:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320743.644020:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320743.644075:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985503104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (389us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320743.644083:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2714 00000100:00100000:0.0:1767320743.644126:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985503104:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320743.644289:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985503232:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320743.644320:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320743.644392:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985503232 00000100:00100000:2.0:1767320743.644396:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985503232 00000100:00100000:2.0:1767320743.644427:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2715 00000100:00100000:2.0:1767320743.644433:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985503232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320743.644449:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320743.644484:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985503232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 51us (164us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320743.644492:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2715 00000100:00100000:1.0:1767320743.644560:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985503232:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320743.772999:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320744.795550:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320745.819641:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320746.843440:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320747.178122:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320747.178201:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920634496 00000100:00100000:0.0:1767320747.178213:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920634496 00000100:00100000:2.0:1767320747.178458:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:3.0:1767320747.178731:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320747.179131:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920634368 00000100:00100000:0.0:1767320747.179137:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920634368 00000100:00100000:0.0:1767320747.179206:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5405 00000100:00100000:0.0:1767320747.179218:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920634496:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320747.179294:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920634496:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (1181us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320747.179302:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5405 00000100:00100000:0.0:1767320747.179333:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6145 00000100:00100000:0.0:1767320747.179338:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920634368:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320747.179355:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320747.179375:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920634368:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 37us (920us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320747.179394:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6145 00000100:00080000:3.0:1767320747.230226:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320747.230237:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320747.230240:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320747.230243:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320747.230248:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320747.230256:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5990 00000100:00080000:3.0:1767320747.230261:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320747.230268:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320747.230310:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5180 x1853163985503360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320747.230343:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320747.230346:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320747.230348:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320747.230351:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5990 00000100:00080000:3.0:1767320747.230353:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320747.230357:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320747.230370:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4e00 x1853163985503488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320747.230408:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320747.230411:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320747.230414:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320747.230417:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5990 00000100:00080000:3.0:1767320747.230420:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320747.230423:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320747.230434:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7b80 x1853163985503744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320747.230456:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320747.230460:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320747.230462:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320747.230465:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5990 00000100:00080000:3.0:1767320747.230467:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320747.230470:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320747.230480:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5c00 x1853163985504000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320747.230527:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985504128:0@lo:400:kworker.0 00000100:00100000:3.0:1767320747.230603:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320747.230658:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320747.230664:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985503872:0@lo:400:kworker.0 00000100:00100000:3.0:1767320747.230688:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320747.231079:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985503360:0@lo:38:kworker.0 00000100:00100000:1.0:1767320747.231111:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320747.231146:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985503488:0@lo:38:kworker.0 00000100:00100000:1.0:1767320747.231198:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320747.231223:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985503744:0@lo:38:kworker.0 00000100:00100000:1.0:1767320747.231243:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320747.231261:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985504000:0@lo:38:kworker.0 00000100:00100000:1.0:1767320747.231279:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320747.231309:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985504256:0@lo:400:kworker.0 00000100:00100000:1.0:1767320747.231331:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320747.231346:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320747.231350:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985503616:0@lo:400:kworker.0 00000100:00100000:1.0:1767320747.231365:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320747.231388:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504128 00000100:00100000:1.0:1767320747.231395:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504128 00000100:00100000:1.0:1767320747.231436:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504256 00000100:00100000:1.0:1767320747.231439:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504256 00000100:00100000:1.0:1767320747.231457:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3505 00000100:00100000:1.0:1767320747.231463:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985504128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320747.231520:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985504128:12345-0@lo:400:kworker.0 Request processed in 56us (917us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320747.231527:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3505 00000100:00100000:1.0:1767320747.231542:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3506 00000100:00100000:1.0:1767320747.231548:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985504256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320747.231586:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985504256:12345-0@lo:400:kworker.0 Request processed in 38us (254us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320747.231591:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3506 00000100:00100000:1.0:1767320747.231605:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985503872 00000100:00100000:1.0:1767320747.231609:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985503872 00000100:00100000:1.0:1767320747.231636:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985503360 00000100:00100000:1.0:1767320747.231639:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985503360 00000100:00100000:1.0:1767320747.231646:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985503744 00000100:00100000:1.0:1767320747.231648:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985503744 00000100:00100000:1.0:1767320747.231654:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504000 00000100:00100000:1.0:1767320747.231656:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504000 00000100:00100000:1.0:1767320747.231661:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985503616 00000100:00100000:1.0:1767320747.231663:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985503616 00000100:00100000:1.0:1767320747.231674:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5406 00000100:00100000:1.0:1767320747.231679:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985503872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320747.231703:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985503872:12345-0@lo:400:kworker.0 Request processed in 24us (1014us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320747.231708:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5406 00000100:00100000:1.0:1767320747.231716:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5407 00000100:00100000:1.0:1767320747.231720:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985503616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320747.231738:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985503616:12345-0@lo:400:kworker.0 Request processed in 18us (372us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320747.231743:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5407 00000100:00100000:1.0:1767320747.231753:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6146 00000100:00100000:1.0:1767320747.231756:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985503360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320747.231771:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320747.231796:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985503360:12345-0@lo:38:kworker.0 Request processed in 38us (685us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320747.231801:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6146 00000100:00100000:1.0:1767320747.231810:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6147 00000100:00100000:1.0:1767320747.231813:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985503744:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320747.231820:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320747.231835:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985503744:12345-0@lo:38:kworker.0 Request processed in 21us (591us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320747.231839:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6147 00000100:00100000:1.0:1767320747.231846:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6148 00000100:00100000:1.0:1767320747.232101:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985504000:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320747.232113:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320747.232141:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985504000:12345-0@lo:38:kworker.0 Request processed in 40us (860us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320747.232229:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985503488 00000100:00100000:2.0:1767320747.232235:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985503488 00000100:00100000:2.0:1767320747.232258:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2027 00000100:00100000:2.0:1767320747.232262:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985503488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320747.232272:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320747.232300:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985503488:12345-0@lo:38:kworker.0 Request processed in 37us (1102us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320747.232306:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2027 00000100:00080000:3.0:1767320747.234773:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320747.234783:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320747.234786:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320747.234847:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985503872:0@lo:400:kworker.0 00000100:00100000:3.0:1767320747.234863:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985504128:0@lo:400:kworker.0 00000100:00080000:3.0:1767320747.234925:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320747.234931:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320747.234934:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985503360:0@lo:38:kworker.0 00000100:00080000:3.0:1767320747.234944:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320747.234947:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320747.234950:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985503488:0@lo:38:kworker.0 00000100:00080000:3.0:1767320747.234959:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320747.234962:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320747.234965:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985503744:0@lo:38:kworker.0 00000100:00080000:3.0:1767320747.234975:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320747.234978:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320747.234981:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985504000:0@lo:38:kworker.0 00000100:00100000:3.0:1767320747.235018:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985503616:0@lo:400:kworker.0 00000100:00100000:3.0:1767320747.235030:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985504256:0@lo:400:kworker.0 00000100:00100000:1.0:1767320747.242071:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6148 00010000:00010000:1.0:1767320747.867398:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320748.763915:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985504384:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320748.763988:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320748.764645:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504384 00000100:00100000:3.0:1767320748.764655:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504384 00000100:00100000:3.0:1767320748.764713:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2716 00000100:00100000:3.0:1767320748.764721:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985504384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320748.764752:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320748.764813:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985504384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (825us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320748.764820:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2716 00000100:00100000:1.0:1767320748.765613:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985504512:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320748.765657:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320748.766329:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504512 00000100:00100000:2.0:1767320748.766338:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504512 00000100:00100000:2.0:1767320748.766384:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2717 00000100:00100000:2.0:1767320748.766393:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985504512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320748.766418:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320748.766484:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985504512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (825us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320748.766491:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2717 00000100:00100000:1.0:1767320748.766692:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985504512:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320748.767715:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985504384:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320748.891567:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320749.915572:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320750.940383:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320751.963686:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320752.304382:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320752.304429:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320752.304533:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920634752 00000100:00100000:1.0:1767320752.304544:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920634752 00000100:00100000:0.0:1767320752.304546:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920634624 00000100:00100000:0.0:1767320752.304553:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920634624 00000100:00100000:0.0:1767320752.304603:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5408 00000100:00100000:1.0:1767320752.304613:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6149 00000100:00100000:0.0:1767320752.304618:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920634752:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320752.304627:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920634624:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320752.304650:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320752.304698:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920634752:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320752.304707:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5408 00000100:00080000:3.0:1767320752.304713:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320752.310537:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920634624:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 5898us (6092us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320752.310555:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6149 00000100:00080000:2.0:1767320752.347322:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.347333:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320752.347337:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.347340:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.347344:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.347352:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.347357:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.347363:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320752.347401:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05f80 x1853163985504640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320752.347456:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985504640:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.347542:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320752.347707:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504640 00000100:00100000:1.0:1767320752.347714:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504640 00000100:00100000:1.0:1767320752.347762:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6150 00000100:00100000:1.0:1767320752.347767:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985504640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320752.347787:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320752.352360:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.352370:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.352375:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.352382:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.352386:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.352397:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320752.352440:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04e00 x1853163985504768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320752.354264:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985504768:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.354348:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320752.354497:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504768 00000100:00100000:0.0:1767320752.354506:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504768 00000100:00100000:0.0:1767320752.354546:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2028 00000100:00100000:0.0:1767320752.354553:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985504768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320752.354574:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320752.354618:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985504768:12345-0@lo:38:kworker.0 Request processed in 65us (269us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320752.354626:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2028 00000100:00100000:1.0:1767320752.356514:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985504640:12345-0@lo:38:kworker.0 Request processed in 8736us (8963us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320752.356530:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6150 00000100:00100000:0.0:1767320752.368234:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985504896:0@lo:400:kworker.0 00000100:00100000:0.0:1767320752.368302:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320752.368397:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985504896 00000100:00100000:1.0:1767320752.368405:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985504896 00000100:00100000:1.0:1767320752.368483:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5409 00000100:00100000:1.0:1767320752.368493:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985504896:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320752.368597:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320752.368604:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320752.368608:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985504640:0@lo:38:kworker.0 00000100:00080000:3.0:1767320752.368618:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320752.368663:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320752.368667:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320752.368670:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985504768:0@lo:38:kworker.0 00000100:00080000:2.0:1767320752.369972:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.369984:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.369988:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.369995:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.370000:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.370006:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320752.370037:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04e00 x1853163985505024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320752.370079:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.370083:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.370085:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.370088:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.370090:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.370104:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320752.370115:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04380 x1853163985505280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320752.370203:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985505536:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.370255:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.370305:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985505024:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.370329:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.370347:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985505280:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.370365:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.370404:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985505152:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.370426:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.370445:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320752.370451:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985505408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.370482:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.370522:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505024 00000100:00100000:3.0:1767320752.370528:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505024 00000100:00100000:3.0:1767320752.370558:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505280 00000100:00100000:3.0:1767320752.370561:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505280 00000100:00100000:3.0:1767320752.370570:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505152 00000100:00100000:3.0:1767320752.370572:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505152 00000100:00100000:3.0:1767320752.370604:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5410 00000100:00100000:3.0:1767320752.370612:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985505152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.370657:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985505152:12345-0@lo:400:kworker.0 Request processed in 45us (229us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320752.370665:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5410 00000100:00100000:3.0:1767320752.370682:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6151 00000100:00100000:3.0:1767320752.370686:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985505024:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320752.370703:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320752.370740:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985505024:12345-0@lo:38:kworker.0 Request processed in 52us (410us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320752.370746:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6151 00000100:00100000:3.0:1767320752.370771:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6152 00000100:00100000:3.0:1767320752.370777:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985505280:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320752.370785:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985504896:12345-0@lo:400:kworker.0 Request processed in 2290us (2478us total) trans 0 rc 0/0 00010000:00020000:3.0:1767320752.370788:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320752.370798:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5409 00000100:00100000:3.0:1767320752.370819:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985505280:12345-0@lo:38:kworker.0 Request processed in 40us (451us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320752.370826:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6152 00000100:00100000:3.0:1767320752.370849:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505536 00000100:00100000:3.0:1767320752.370854:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505536 00000100:00100000:3.0:1767320752.370882:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505408 00000100:00100000:3.0:1767320752.370885:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505408 00000100:00100000:3.0:1767320752.370907:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3507 00000100:00100000:3.0:1767320752.370912:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985505536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320752.370920:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3508 00000100:00100000:1.0:1767320752.370932:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985505408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.370946:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985505536:12345-0@lo:400:kworker.0 Request processed in 32us (689us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320752.370954:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3507 00000100:00080000:3.0:1767320752.370970:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320752.370974:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320752.370979:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985505408:12345-0@lo:400:kworker.0 Request processed in 47us (493us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320752.370979:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320752.370989:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3508 00000100:00100000:3.0:1767320752.371008:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985505408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.371023:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985505152:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.371056:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985505536:0@lo:400:kworker.0 00000100:00080000:3.0:1767320752.371075:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320752.371079:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320752.371083:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985505024:0@lo:38:kworker.0 00000100:00080000:3.0:1767320752.371090:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320752.371093:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320752.371095:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985505280:0@lo:38:kworker.0 00000100:00100000:0.0:1767320752.371767:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985504896:0@lo:400:kworker.0 00010000:00010000:1.0:1767320752.989387:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320753.883880:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320753.883899:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985505664:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320753.883951:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985505792:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320753.883970:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320753.884001:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320753.884047:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505664 00000100:00100000:3.0:1767320753.884054:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505664 00000100:00100000:2.0:1767320753.884088:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505792 00000100:00100000:2.0:1767320753.884095:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505792 00000100:00100000:3.0:1767320753.884108:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2718 00000100:00100000:3.0:1767320753.884119:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985505664:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320753.884143:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2719 00002000:00000024:3.0:1767320753.884150:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320753.884153:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985505792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320753.884201:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320753.884221:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985505664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (250us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320753.884230:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2718 00000100:00100000:2.0:1767320753.884264:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985505792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (262us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320753.884272:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2719 00000100:00100000:3.0:1767320753.884305:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985505792:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320753.885066:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985505664:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320754.013950:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320755.035878:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320756.059454:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320757.083446:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320757.424706:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320757.424759:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320757.424929:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920635008 00000100:00100000:1.0:1767320757.424939:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920635008 00000100:00100000:1.0:1767320757.425013:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920634880 00000100:00100000:1.0:1767320757.425017:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920634880 00000100:00100000:1.0:1767320757.425040:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5411 00000100:00100000:1.0:1767320757.425050:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920635008:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320757.425124:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920635008:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (429us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320757.425132:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5411 00000100:00100000:1.0:1767320757.425149:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6153 00000100:00100000:1.0:1767320757.425155:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920634880:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320757.425206:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320757.425233:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920634880:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 77us (472us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320757.425239:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6153 00000100:00080000:3.0:1767320757.425314:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320757.467453:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320757.467543:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320757.467546:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320757.467550:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320757.467555:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320757.467564:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6000 00000100:00080000:1.0:1767320757.467568:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320757.467575:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320757.467619:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985505920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320757.467679:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985505920:0@lo:38:kworker.0 00000100:00100000:3.0:1767320757.467758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320757.468027:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320757.468031:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320757.468034:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320757.468037:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6000 00000100:00080000:1.0:1767320757.468040:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320757.468044:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320757.468059:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985506048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320757.468406:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985505920 00000100:00100000:2.0:1767320757.468415:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985505920 00000100:00100000:2.0:1767320757.468498:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6154 00000100:00100000:2.0:1767320757.468506:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985505920:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320757.468527:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320757.468608:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985506048:0@lo:38:kworker.0 00000100:00100000:3.0:1767320757.468638:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320757.468906:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320757.468910:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320757.468912:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320757.468917:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6000 00000100:00080000:1.0:1767320757.468919:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320757.468923:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320757.468947:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612df80 x1853163985506304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320757.468977:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320757.468980:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320757.468982:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320757.468986:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6000 00000100:00080000:1.0:1767320757.468988:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320757.468992:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320757.469005:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c380 x1853163985506560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320757.469050:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985506432:0@lo:400:kworker.0 00000100:00100000:1.0:1767320757.469079:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320757.469104:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320757.469110:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985506688:0@lo:400:kworker.0 00000100:00100000:1.0:1767320757.469133:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320757.469160:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506688 00000100:00100000:1.0:1767320757.469179:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506688 00000100:00100000:1.0:1767320757.469223:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3509 00000100:00100000:1.0:1767320757.469230:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985506688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.469369:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985506176:0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.469406:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320757.469427:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320757.469432:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985506816:0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.469452:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320757.469496:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506816 00000100:00100000:0.0:1767320757.469500:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506816 00000100:00100000:0.0:1767320757.469527:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3510 00000100:00100000:0.0:1767320757.469533:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985506816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.469567:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985506816:12345-0@lo:400:kworker.0 Request processed in 34us (113us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320757.469574:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3510 00000100:00100000:0.0:1767320757.469614:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985506816:0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.469647:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506432 00000100:00100000:0.0:1767320757.469650:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506432 00000100:00100000:0.0:1767320757.469675:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506176 00000100:00100000:0.0:1767320757.469678:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506176 00000100:00100000:0.0:1767320757.469711:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5412 00000100:00100000:0.0:1767320757.469717:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985506432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320757.469884:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985506688:12345-0@lo:400:kworker.0 Request processed in 653us (748us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320757.469891:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3509 00000100:00100000:1.0:1767320757.469915:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5413 00000100:00100000:1.0:1767320757.469923:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985506176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320757.469956:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985506176:12345-0@lo:400:kworker.0 Request processed in 35us (549us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320757.469962:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5413 00000100:00100000:1.0:1767320757.469988:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985506176:0@lo:400:kworker.0 00000100:00100000:2.0:1767320757.470595:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985505920:12345-0@lo:38:kworker.0 Request processed in 2086us (2833us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320757.470603:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6154 00000100:00100000:2.0:1767320757.470657:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506048 00000100:00100000:2.0:1767320757.470662:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506048 00000100:00100000:0.0:1767320757.470780:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985506432:12345-0@lo:400:kworker.0 Request processed in 1063us (1700us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320757.470787:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5412 00000100:00100000:2.0:1767320757.470838:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2029 00000100:00100000:2.0:1767320757.470844:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985506048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320757.470861:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320757.470892:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985506048:12345-0@lo:38:kworker.0 Request processed in 48us (2254us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767320757.470954:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320757.470960:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320757.470963:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985505920:0@lo:38:kworker.0 00000100:00100000:3.0:1767320757.470985:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985506304:0@lo:38:kworker.0 00000100:00100000:3.0:1767320757.471011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320757.471029:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985506560:0@lo:38:kworker.0 00000100:00100000:3.0:1767320757.471044:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320757.471060:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320757.471063:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320757.471066:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985506048:0@lo:38:kworker.0 00000100:00100000:3.0:1767320757.471085:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506304 00000100:00100000:3.0:1767320757.471088:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506304 00000100:00100000:3.0:1767320757.471105:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506560 00000100:00100000:3.0:1767320757.471107:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506560 00000100:00100000:3.0:1767320757.471119:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6155 00000100:00100000:3.0:1767320757.471123:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985506304:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320757.471132:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320757.471155:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985506304:12345-0@lo:38:kworker.0 Request processed in 31us (144us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320757.471160:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6155 00000100:00100000:3.0:1767320757.471184:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6156 00000100:00100000:3.0:1767320757.471187:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985506560:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320757.471195:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320757.471214:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985506560:12345-0@lo:38:kworker.0 Request processed in 25us (168us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320757.471218:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6156 00000100:00080000:3.0:1767320757.471233:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320757.471237:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320757.471239:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320757.471253:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320757.471257:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320757.471259:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985506304:0@lo:38:kworker.0 00000100:00080000:3.0:1767320757.471277:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320757.471280:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320757.471283:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985506560:0@lo:38:kworker.0 00000100:00100000:1.0:1767320757.471593:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985506688:0@lo:400:kworker.0 00000100:00100000:1.0:1767320757.471609:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985506432:0@lo:400:kworker.0 00000100:00100000:2.0:1767320757.471791:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2029 00010000:00010000:1.0:1767320758.108922:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320759.003493:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985506944:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320759.003563:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320759.003654:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985506944 00000100:00100000:0.0:1767320759.003662:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985506944 00000100:00100000:0.0:1767320759.003725:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2720 00000100:00100000:0.0:1767320759.003735:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985506944:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320759.003763:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320759.003894:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320759.003904:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985507072:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320759.003929:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320759.003952:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507072 00000100:00100000:2.0:1767320759.003967:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507072 00000100:00100000:2.0:1767320759.003992:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2721 00000100:00100000:2.0:1767320759.003999:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985507072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320759.004014:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320759.004041:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985507072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (111us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320759.004048:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2721 00000100:00080000:2.0:1767320759.004069:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320759.004072:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320759.004099:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985507072:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320759.005771:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985506944:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2030us (2201us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320759.005782:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2720 00000100:00100000:1.0:1767320759.006360:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985506944:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320759.132638:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320760.155400:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320761.179791:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320762.205860:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320762.540276:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320762.540376:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320762.540413:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920635264 00000100:00100000:2.0:1767320762.540422:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920635264 00000100:00100000:2.0:1767320762.540475:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920635136 00000100:00100000:2.0:1767320762.540477:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920635136 00000100:00100000:2.0:1767320762.540496:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5414 00000100:00100000:2.0:1767320762.540504:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920635264:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320762.540628:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920635264:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 124us (361us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320762.540636:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5414 00000100:00100000:2.0:1767320762.540659:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6157 00000100:00100000:2.0:1767320762.540665:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920635136:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320762.540687:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320762.540717:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920635136:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (341us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320762.540723:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6157 00000100:00080000:3.0:1767320762.540793:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320762.587619:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320762.587631:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320762.587634:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320762.587638:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320762.587644:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320762.587654:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6005 00000100:00080000:0.0:1767320762.587658:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320762.587664:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320762.587707:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89f80 x1853163985507200/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320762.587771:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985507200:0@lo:38:kworker.0 00000100:00100000:3.0:1767320762.587838:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320762.587955:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507200 00000100:00100000:1.0:1767320762.587963:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507200 00000100:00100000:1.0:1767320762.588001:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6158 00000100:00100000:1.0:1767320762.588008:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985507200:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320762.588029:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320762.588111:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320762.588115:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320762.588117:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320762.588121:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6005 00000100:00080000:0.0:1767320762.588123:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320762.588127:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320762.588140:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b800 x1853163985507328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320762.588985:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985507328:0@lo:38:kworker.0 00000100:00100000:3.0:1767320762.589026:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320762.589142:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985507200:12345-0@lo:38:kworker.0 Request processed in 1133us (1304us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320762.589159:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6158 00000100:00080000:0.0:1767320762.590134:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320762.590139:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320762.590142:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320762.590147:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6005 00000100:00080000:0.0:1767320762.590151:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320762.590156:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320762.590185:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985507584/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320762.590210:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320762.590213:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320762.590215:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320762.590218:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6005 00000100:00080000:0.0:1767320762.590221:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320762.590224:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320762.590231:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfbb80 x1853163985507840/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320762.590268:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985507712:0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.590324:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320762.590352:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320762.590356:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985507968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.590375:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320762.590397:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985508096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.590414:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320762.590428:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320762.590431:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985507456:0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.590446:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320762.590468:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507712 00000100:00100000:0.0:1767320762.590474:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507712 00000100:00100000:0.0:1767320762.590518:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507456 00000100:00100000:0.0:1767320762.590520:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507456 00000100:00100000:0.0:1767320762.590537:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5415 00000100:00100000:0.0:1767320762.590544:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985507712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.590625:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985507712:12345-0@lo:400:kworker.0 Request processed in 82us (300us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320762.590632:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5415 00000100:00100000:0.0:1767320762.590645:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5416 00000100:00100000:0.0:1767320762.590651:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985507456:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.590672:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985507456:12345-0@lo:400:kworker.0 Request processed in 21us (225us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320762.590677:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5416 00000100:00100000:0.0:1767320762.590705:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985507456:0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.590725:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985507712:0@lo:400:kworker.0 00000100:00080000:0.0:1767320762.590761:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320762.592200:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507328 00000100:00100000:2.0:1767320762.592215:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507328 00000100:00100000:2.0:1767320762.592268:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2030 00000100:00100000:2.0:1767320762.592290:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985507328:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320762.592314:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320762.592378:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985507328:12345-0@lo:38:kworker.0 Request processed in 88us (3351us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320762.592385:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2030 00000100:00100000:1.0:1767320762.592539:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507968 00000100:00100000:1.0:1767320762.592544:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507968 00000100:00100000:1.0:1767320762.592599:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985508096 00000100:00100000:1.0:1767320762.592602:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985508096 00000100:00100000:1.0:1767320762.592623:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3511 00000100:00100000:1.0:1767320762.592629:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985507968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320762.592671:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985507968:12345-0@lo:400:kworker.0 Request processed in 42us (2294us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320762.592676:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3511 00000100:00100000:1.0:1767320762.592689:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3512 00000100:00100000:1.0:1767320762.592694:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985508096:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320762.592724:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985508096:12345-0@lo:400:kworker.0 Request processed in 30us (2309us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320762.592729:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3512 00000100:00100000:1.0:1767320762.592783:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985507968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320762.592810:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985508096:0@lo:400:kworker.0 00000100:00080000:2.0:1767320762.592960:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320762.592964:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320762.593509:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320762.593516:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320762.593520:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985507200:0@lo:38:kworker.0 00000100:00100000:3.0:1767320762.593546:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985507584:0@lo:38:kworker.0 00000100:00100000:3.0:1767320762.593589:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320762.593621:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985507840:0@lo:38:kworker.0 00000100:00100000:3.0:1767320762.593643:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320762.593743:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320762.593747:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320762.593751:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985507328:0@lo:38:kworker.0 00000100:00100000:0.0:1767320762.598757:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507584 00000100:00100000:0.0:1767320762.598773:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507584 00000100:00100000:0.0:1767320762.598845:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985507840 00000100:00100000:0.0:1767320762.598850:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985507840 00000100:00100000:0.0:1767320762.598890:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6159 00000100:00100000:0.0:1767320762.598902:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985507584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320762.598931:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320762.598949:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6160 00000100:00100000:3.0:1767320762.598956:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985507840:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320762.598967:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320762.603874:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985507584:12345-0@lo:38:kworker.0 Request processed in 4960us (10270us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320762.603890:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6159 00000100:00080000:2.0:1767320762.604087:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320762.604093:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320762.604098:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985507584:0@lo:38:kworker.0 00000100:00100000:3.0:1767320762.606456:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985507840:12345-0@lo:38:kworker.0 Request processed in 7488us (12800us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320762.606470:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6160 00000100:00080000:2.0:1767320762.606623:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320762.606630:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320762.606634:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985507840:0@lo:38:kworker.0 00010000:00010000:1.0:1767320763.227753:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320764.126332:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320764.126357:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985508224:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320764.126433:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320764.126529:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985508224 00000100:00100000:2.0:1767320764.126544:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985508224 00000100:00080000:3.0:1767320764.126579:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320764.126598:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2722 00000100:00100000:2.0:1767320764.126608:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985508224:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320764.126622:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985508352:0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320764.126638:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320764.126651:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320764.126688:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985508224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (245us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320764.126696:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2722 00000100:00100000:1.0:1767320764.126704:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985508352 00000100:00100000:1.0:1767320764.126707:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985508352 00000100:00100000:1.0:1767320764.126730:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2723 00000100:00100000:1.0:1767320764.126737:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985508352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320764.126752:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320764.126791:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985508352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320764.126801:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2723 00000100:00080000:3.0:1767320764.126856:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320764.127107:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985508352:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320764.128775:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985508224:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320764.251528:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320765.275433:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320766.301286:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767320766.633826:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920635392 offset 0 length 520: 4 00000100:00100000:3.0:1767320766.634014:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320766.634051:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320766.634102:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920635520 00000100:00100000:2.0:1767320766.634111:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920635520 00000100:00100000:2.0:1767320766.634302:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920635648 00000100:00100000:2.0:1767320766.634307:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920635648 00000100:00080000:1.0:1767320766.634371:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320766.634468:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5417 00000100:00100000:0.0:1767320766.634480:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920635648:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320766.634524:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6161 00000100:00100000:3.0:1767320766.634532:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920635520:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320766.634561:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320766.634618:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920635520:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 85us (605us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320766.634625:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6161 00000100:00100000:0.0:1767320766.635134:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920635648:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 655us (1080us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320766.635290:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5417 00010000:00010000:2.0:1767320767.323898:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320767.708280:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320742/real 1767320742] req@ffff8ce1a612e680 x1853163985501952/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320767 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320767.708315:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a612e680 x1853163985501952/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320767 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320767.708334:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320767.708351:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320767.708357:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00080000:2.0:1767320767.708360:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1767320767.708363:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985501952:0@lo:250:kworker.0 00000100:00080000:2.0:1767320767.708366:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320767.708373:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 5990 00000100:00080000:2.0:1767320767.708377:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320767.708384:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320767.712334:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579880 x1853163985508480/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320767.712462:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320767.712467:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320767.712470:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320767.712475:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320767.712479:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320767.712490:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320767.712761:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357bb80 x1853163985508608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320767.712772:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320767.712775:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320767.712776:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320767.712779:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320767.712782:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320767.712785:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320767.712796:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579500 x1853163985508736/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320767.712887:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985508864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320767.712958:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320767.713078:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985508864 00000100:00100000:1.0:1767320767.713087:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985508864 00000100:00100000:1.0:1767320767.713237:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5418 00000100:00080000:2.0:1767320767.713241:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320767.713246:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320767.713249:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320767.713251:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985508864:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320767.713255:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320767.713260:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320767.713264:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320767.713290:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578700 x1853163985508992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320767.713319:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320767.713322:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320767.713324:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320767.713328:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320767.713331:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320767.713334:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320767.713345:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985509248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320767.713350:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985508480:0@lo:250:kworker.0 00000400:00000100:3.0:1767320767.713383:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985508480 offset 0 length 520: 4 00000100:00100000:3.0:1767320767.713409:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985508608:0@lo:38:kworker.0 00000100:00100000:3.0:1767320767.713430:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320767.713445:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985508736:0@lo:38:kworker.0 00000100:00100000:3.0:1767320767.713466:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320767.713483:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985508992:0@lo:38:kworker.0 00000100:00100000:3.0:1767320767.713503:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320767.713625:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985509248:0@lo:38:kworker.0 00000100:00100000:2.0:1767320767.713673:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985509504:0@lo:400:kworker.0 00000100:00100000:1.0:1767320767.713674:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985508864:12345-0@lo:400:kworker.0 Request processed in 426us (714us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320767.713687:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5418 00000100:00100000:1.0:1767320767.713705:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985508608 00000100:00100000:1.0:1767320767.713709:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985508608 00000100:00100000:2.0:1767320767.713714:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320767.713734:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985508992 00000100:00100000:1.0:1767320767.713738:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985508992 00000100:00100000:1.0:1767320767.713756:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6162 00000100:00100000:1.0:1767320767.713761:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985508608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320767.713779:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320767.713787:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985508736 00000100:00100000:2.0:1767320767.713793:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985508736 00000100:00100000:1.0:1767320767.713810:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985508608:12345-0@lo:38:kworker.0 Request processed in 47us (378us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320767.713820:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6162 00000100:00100000:2.0:1767320767.713820:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2031 00000100:00100000:2.0:1767320767.713830:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985508736:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320767.713839:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6163 00000100:00100000:1.0:1767320767.713844:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985508992:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320767.713854:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320767.713858:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320767.713895:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985508736:12345-0@lo:38:kworker.0 Request processed in 63us (426us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320767.713897:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985508992:12345-0@lo:38:kworker.0 Request processed in 50us (392us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320767.713907:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2031 00000100:00100000:1.0:1767320767.713911:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6163 00000100:00100000:1.0:1767320767.713936:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985509504 00000100:00100000:1.0:1767320767.713940:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985509504 00000100:00100000:2.0:1767320767.713972:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985509376:0@lo:400:kworker.0 00000100:00100000:1.0:1767320767.713973:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3513 00000100:00100000:1.0:1767320767.713983:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985509504:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320767.714021:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985509504:12345-0@lo:400:kworker.0 Request processed in 39us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320767.714027:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3513 00000100:00100000:1.0:1767320767.714068:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985509504:0@lo:400:kworker.0 00000100:00100000:1.0:1767320767.714095:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985509120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320767.714118:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320767.714240:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320767.714269:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320767.714273:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320767.714749:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985508864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320767.714782:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985509376 00000100:00100000:0.0:1767320767.714787:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985509376 00000100:00100000:0.0:1767320767.714876:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3514 00000100:00100000:0.0:1767320767.714884:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985509376:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320767.714930:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985509376:12345-0@lo:400:kworker.0 Request processed in 47us (691us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320767.714936:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3514 00000100:00100000:0.0:1767320767.714962:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985509376:0@lo:400:kworker.0 00000100:00080000:2.0:1767320767.715266:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320767.715280:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320767.715311:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320767.715317:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320767.715322:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985508608:0@lo:38:kworker.0 00000100:00100000:2.0:1767320767.715324:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985509120 00000100:00100000:2.0:1767320767.715330:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985509120 00000100:00080000:3.0:1767320767.715333:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320767.715339:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320767.715343:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985508736:0@lo:38:kworker.0 00000100:00080000:3.0:1767320767.715351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320767.715354:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320767.715357:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985508992:0@lo:38:kworker.0 00000100:00100000:2.0:1767320767.715417:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5419 00000100:00100000:2.0:1767320767.715425:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985509120:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320767.715463:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985509120:12345-0@lo:400:kworker.0 Request processed in 39us (1344us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320767.715470:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5419 00000100:00100000:1.0:1767320767.715481:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985509248 00000100:00100000:1.0:1767320767.715487:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985509248 00000100:00100000:1.0:1767320767.715513:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6164 00000100:00100000:1.0:1767320767.715518:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985509248:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320767.715527:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985509120:0@lo:400:kworker.0 00010000:00020000:1.0:1767320767.715537:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320767.715575:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985509248:12345-0@lo:38:kworker.0 Request processed in 55us (296us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320767.715583:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6164 00000100:00080000:2.0:1767320767.715587:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320767.715593:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320767.715598:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985509248:0@lo:38:kworker.0 00000100:00080000:0.0:1767320767.715753:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320768.347578:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320769.243597:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985509632:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320769.243671:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320769.243900:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320769.243910:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985509760:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320769.243938:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320769.244063:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985509632 00000100:00100000:1.0:1767320769.244071:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985509632 00000100:00100000:1.0:1767320769.244130:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985509760 00000100:00100000:1.0:1767320769.244134:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985509760 00000100:00100000:1.0:1767320769.244200:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2724 00000100:00100000:1.0:1767320769.244210:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985509632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320769.244238:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320769.244332:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985509632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 124us (661us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320769.244339:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2724 00000100:00100000:1.0:1767320769.244355:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2725 00000100:00100000:1.0:1767320769.244361:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985509760:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320769.244376:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320769.244403:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985509760:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (464us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320769.244409:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2725 00000100:00100000:3.0:1767320769.244503:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985509760:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320769.244725:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320769.244730:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320769.244753:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985509632:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320769.371554:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320770.395400:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320771.419501:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320771.754522:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320771.754572:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320771.754670:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920635776 00000100:00100000:2.0:1767320771.754682:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920635776 00000100:00100000:2.0:1767320771.754730:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920635904 00000100:00100000:2.0:1767320771.754732:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920635904 00000100:00100000:2.0:1767320771.754773:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5420 00000100:00100000:2.0:1767320771.754782:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920635904:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767320771.754847:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320771.754867:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920635904:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320771.754877:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5420 00000100:00100000:2.0:1767320771.754909:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6165 00000100:00100000:2.0:1767320771.754913:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920635776:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320771.754932:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320771.754956:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920635776:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (446us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320771.754962:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6165 00010000:00010000:1.0:1767320772.445680:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320772.827484:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320772.827495:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320772.827498:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320772.827502:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320772.827507:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320772.827527:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6015 00000100:00080000:3.0:1767320772.827531:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320772.827538:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320772.827575:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05f80 x1853163985509888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320772.827601:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320772.827605:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320772.827607:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320772.827610:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6015 00000100:00080000:3.0:1767320772.827612:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320772.827615:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320772.827633:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18fb80 x1853163985510016/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320772.827668:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320772.827672:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320772.827673:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320772.827677:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6015 00000100:00080000:3.0:1767320772.827680:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320772.827683:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320772.827693:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ea00 x1853163985510272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320772.827710:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985510144:0@lo:400:kworker.0 00000100:00080000:3.0:1767320772.827720:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320772.827723:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320772.827725:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320772.827727:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6015 00000100:00080000:3.0:1767320772.827729:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320772.827732:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320772.827741:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18e300 x1853163985510528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320772.827776:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320772.827795:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985510400:0@lo:400:kworker.0 00000100:00100000:0.0:1767320772.827877:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985510784:0@lo:400:kworker.0 00000100:00100000:0.0:1767320772.827907:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320772.827973:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510784 00000100:00100000:0.0:1767320772.827981:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510784 00000100:00100000:0.0:1767320772.828044:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3515 00000100:00100000:0.0:1767320772.828053:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985510784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320772.828109:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985510784:12345-0@lo:400:kworker.0 Request processed in 57us (200us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320772.828116:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3515 00000100:00080000:0.0:1767320772.828135:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320772.828678:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985509888:0@lo:38:kworker.0 00000100:00100000:0.0:1767320772.828766:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985510784:0@lo:400:kworker.0 00000100:00100000:1.0:1767320772.830887:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985510656:0@lo:400:kworker.0 00000100:00100000:1.0:1767320772.830946:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320772.834284:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510656 00000100:00100000:0.0:1767320772.834300:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510656 00000100:00100000:0.0:1767320772.834386:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3516 00000100:00100000:0.0:1767320772.834396:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985510656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320772.834497:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985510656:12345-0@lo:400:kworker.0 Request processed in 103us (3550us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320772.834503:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3516 00000100:00080000:1.0:1767320772.835065:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320772.835123:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985510656:0@lo:400:kworker.0 00000100:00100000:3.0:1767320772.837344:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510144 00000100:00100000:3.0:1767320772.837361:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510144 00000100:00100000:3.0:1767320772.837462:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5421 00000100:00100000:3.0:1767320772.837474:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985510144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320772.837565:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985510144:12345-0@lo:400:kworker.0 Request processed in 96us (9788us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320772.837573:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5421 00000100:00100000:3.0:1767320772.837609:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320772.837653:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510400 00000100:00100000:3.0:1767320772.837657:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510400 00000100:00100000:3.0:1767320772.837677:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5422 00000100:00100000:3.0:1767320772.837683:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985510400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320772.837713:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985510400:12345-0@lo:400:kworker.0 Request processed in 30us (103us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320772.837719:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5422 00000100:00100000:3.0:1767320772.837765:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985510400:0@lo:400:kworker.0 00000100:00080000:1.0:1767320772.838962:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320772.839007:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985510144:0@lo:400:kworker.0 00000100:00100000:2.0:1767320772.840037:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320772.840112:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985510016:0@lo:38:kworker.0 00000100:00100000:2.0:1767320772.840473:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320772.840508:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985510272:0@lo:38:kworker.0 00000100:00100000:2.0:1767320772.840533:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320772.840556:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985510528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320772.840575:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320772.840614:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985509888 00000100:00100000:2.0:1767320772.840621:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985509888 00000100:00100000:2.0:1767320772.840651:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510272 00000100:00100000:2.0:1767320772.840654:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510272 00000100:00100000:2.0:1767320772.840666:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510528 00000100:00100000:2.0:1767320772.840668:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510528 00000100:00100000:2.0:1767320772.840681:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6166 00000100:00100000:2.0:1767320772.840686:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985509888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320772.840705:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320772.840756:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985509888:12345-0@lo:38:kworker.0 Request processed in 68us (723us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320772.840762:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6166 00000100:00100000:0.0:1767320772.840820:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6167 00000100:00100000:0.0:1767320772.840825:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985510272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320772.840835:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320772.840864:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985510272:12345-0@lo:38:kworker.0 Request processed in 37us (331us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320772.840870:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6167 00000100:00100000:0.0:1767320772.840884:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6168 00000100:00100000:0.0:1767320772.840887:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985510528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320772.840897:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320772.840918:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985510528:12345-0@lo:38:kworker.0 Request processed in 29us (342us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320772.840923:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6168 00000100:00080000:2.0:1767320772.840978:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320772.840984:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320772.840987:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985509888:0@lo:38:kworker.0 00000100:00080000:2.0:1767320772.840996:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320772.840999:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320772.841002:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985510272:0@lo:38:kworker.0 00000100:00080000:2.0:1767320772.841008:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320772.841012:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320772.841015:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985510528:0@lo:38:kworker.0 00000100:00100000:3.0:1767320772.841648:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510016 00000100:00100000:3.0:1767320772.841655:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510016 00000100:00100000:3.0:1767320772.841702:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2032 00000100:00100000:3.0:1767320772.841709:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985510016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320772.841722:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320772.842051:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985510016:12345-0@lo:38:kworker.0 Request processed in 340us (1576us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320772.842059:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2032 00000100:00080000:0.0:1767320772.842948:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320772.842956:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320772.842960:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985510016:0@lo:38:kworker.0 00010000:00010000:2.0:1767320773.471127:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320774.363507:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985510912:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320774.363578:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320774.363695:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320774.363704:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985511040:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320774.363732:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320774.363811:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985510912 00000100:00100000:2.0:1767320774.363819:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985510912 00000100:00100000:2.0:1767320774.363880:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511040 00000100:00100000:2.0:1767320774.363883:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511040 00000100:00100000:0.0:1767320774.363915:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2726 00000100:00100000:2.0:1767320774.363923:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2727 00000100:00100000:0.0:1767320774.363930:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985510912:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320774.363937:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985511040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320774.363965:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:2.0:1767320774.363966:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320774.364020:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985511040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320774.364028:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985510912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 102us (447us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320774.364031:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2727 00000100:00100000:0.0:1767320774.364041:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2726 00000100:00100000:0.0:1767320774.364105:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985510912:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320774.364108:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985511040:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320774.364319:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320774.364323:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767320774.491629:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320775.515533:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320776.542107:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320776.873959:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320776.874066:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636160 00000100:00100000:1.0:1767320776.874077:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636160 00000100:00100000:1.0:1767320776.874253:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5423 00000100:00100000:1.0:1767320776.874265:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920636160:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320776.874343:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920636160:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320776.874350:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5423 00000100:00080000:0.0:1767320776.874635:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320776.878045:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320776.878344:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636032 00000100:00100000:2.0:1767320776.878356:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636032 00000100:00100000:2.0:1767320776.878401:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6169 00000100:00100000:2.0:1767320776.878408:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920636032:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320776.878433:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320776.878520:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920636032:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 110us (493us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320776.879052:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6169 00010000:00010000:1.0:1767320777.563620:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320777.948248:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320777.948265:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320777.948269:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320777.948272:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320777.948277:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320777.948285:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6021 00000100:00080000:2.0:1767320777.948290:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320777.948296:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320777.948330:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985511168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320777.948358:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320777.948362:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320777.948363:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320777.948366:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6021 00000100:00080000:2.0:1767320777.948369:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320777.948372:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320777.948387:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578700 x1853163985511296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320777.948414:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320777.948418:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320777.948420:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320777.948423:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6021 00000100:00080000:2.0:1767320777.948426:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320777.948429:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320777.948440:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357bb80 x1853163985511552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320777.948463:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320777.948465:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320777.948467:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320777.948471:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6021 00000100:00080000:2.0:1767320777.948473:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320777.948476:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320777.948485:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579c00 x1853163985511808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320777.948518:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985511424:0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.948580:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320777.948637:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320777.948643:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985512064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.948670:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320777.948700:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985511936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.948723:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320777.948742:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320777.948746:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985511680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.948763:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320777.948790:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511424 00000100:00100000:2.0:1767320777.948799:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511424 00000100:00100000:2.0:1767320777.948851:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511680 00000100:00100000:2.0:1767320777.948854:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511680 00000100:00100000:2.0:1767320777.948876:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5424 00000100:00100000:2.0:1767320777.948883:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985511424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.948930:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985511424:12345-0@lo:400:kworker.0 Request processed in 46us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320777.948937:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5424 00000100:00100000:2.0:1767320777.948955:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5425 00000100:00100000:2.0:1767320777.948962:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985511680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.948990:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985511680:12345-0@lo:400:kworker.0 Request processed in 30us (225us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320777.949005:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5425 00000100:00100000:2.0:1767320777.949025:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512064 00000100:00100000:2.0:1767320777.949028:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512064 00000100:00100000:2.0:1767320777.949062:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511936 00000100:00100000:2.0:1767320777.949066:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511936 00000100:00100000:2.0:1767320777.949085:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3517 00000100:00100000:2.0:1767320777.949091:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985512064:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.949120:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985512064:12345-0@lo:400:kworker.0 Request processed in 29us (450us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320777.949126:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3517 00000100:00100000:2.0:1767320777.949136:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3518 00000100:00100000:2.0:1767320777.949143:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985511936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.949204:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985511936:12345-0@lo:400:kworker.0 Request processed in 62us (480us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320777.949210:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3518 00000100:00080000:2.0:1767320777.949228:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320777.949232:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320777.949234:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320777.949272:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985511680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.949281:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985511936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.949307:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985512064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320777.949315:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985511424:0@lo:400:kworker.0 00000100:00100000:3.0:1767320777.951073:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985511168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320777.951140:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320777.951482:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511168 00000100:00100000:0.0:1767320777.951490:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511168 00000100:00100000:0.0:1767320777.951538:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6170 00000100:00100000:0.0:1767320777.951547:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985511168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320777.951571:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320777.953579:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985511168:12345-0@lo:38:kworker.0 Request processed in 2022us (2428us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320777.953593:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6170 00000100:00080000:3.0:1767320777.953686:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320777.953692:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320777.953696:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985511168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320777.953708:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985511296:0@lo:38:kworker.0 00000100:00100000:3.0:1767320777.953755:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320777.953835:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985511552:0@lo:38:kworker.0 00000100:00100000:3.0:1767320777.953858:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320777.953881:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985511808:0@lo:38:kworker.0 00000100:00100000:3.0:1767320777.953900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320777.954078:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511552 00000100:00100000:1.0:1767320777.954087:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511552 00000100:00100000:1.0:1767320777.954124:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511808 00000100:00100000:1.0:1767320777.954127:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511808 00000100:00100000:1.0:1767320777.954145:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6171 00000100:00100000:1.0:1767320777.954152:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985511552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320777.954202:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320777.954252:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985511552:12345-0@lo:38:kworker.0 Request processed in 98us (392us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320777.954258:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6171 00000100:00100000:1.0:1767320777.954292:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6172 00000100:00100000:1.0:1767320777.954296:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985511808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320777.954306:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320777.954330:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985511808:12345-0@lo:38:kworker.0 Request processed in 33us (428us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320777.954338:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6172 00000100:00080000:3.0:1767320777.955586:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320777.955598:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320777.955602:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985511552:0@lo:38:kworker.0 00000100:00080000:3.0:1767320777.955614:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320777.955617:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320777.955619:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985511808:0@lo:38:kworker.0 00000100:00100000:2.0:1767320777.958457:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985511296 00000100:00100000:2.0:1767320777.958474:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985511296 00000100:00100000:2.0:1767320777.958591:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2033 00000100:00100000:2.0:1767320777.958600:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985511296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320777.958624:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320777.958725:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985511296:12345-0@lo:38:kworker.0 Request processed in 125us (4969us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320777.958732:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2033 00000100:00080000:0.0:1767320777.958804:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320777.958813:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320777.958817:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985511296:0@lo:38:kworker.0 00010000:00010000:1.0:1767320778.587629:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320779.483570:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320779.483579:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985512192:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320779.483594:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985512320:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320779.483659:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320779.483774:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320779.484115:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512192 00000100:00100000:3.0:1767320779.484123:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512192 00000100:00100000:3.0:1767320779.484225:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512320 00000100:00100000:3.0:1767320779.484228:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512320 00000100:00100000:3.0:1767320779.484252:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2728 00000100:00100000:3.0:1767320779.484261:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985512192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320779.484291:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320779.484336:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985512192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (676us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320779.484344:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2728 00000100:00100000:3.0:1767320779.484360:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2729 00000100:00100000:3.0:1767320779.484367:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985512320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320779.484381:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320779.484409:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985512320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (634us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320779.484415:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2729 00000100:00100000:1.0:1767320779.484583:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985512320:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320779.484646:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320779.484651:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320779.484722:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985512192:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320779.611359:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320780.635585:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320781.660788:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320781.993867:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320781.994716:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636288 00000100:00100000:0.0:1767320781.994730:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636288 00000100:00100000:0.0:1767320781.994788:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6173 00000100:00100000:0.0:1767320781.994796:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920636288:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320781.994822:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320781.994901:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920636288:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 104us (1046us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320781.994909:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6173 00000100:00100000:3.0:1767320781.995800:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320781.995870:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636416 00000100:00100000:1.0:1767320781.995876:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636416 00000100:00100000:1.0:1767320781.995939:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5426 00000100:00100000:1.0:1767320781.995948:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920636416:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320781.996020:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920636416:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (224us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320781.996028:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5426 00000100:00080000:0.0:1767320781.996575:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320782.692132:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320783.068384:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320783.068394:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320783.068397:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320783.068401:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320783.068406:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320783.068413:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6026 00000100:00080000:0.0:1767320783.068418:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320783.068424:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320783.068480:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985512448/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320783.068494:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320783.068496:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320783.068498:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320783.068500:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6026 00000100:00080000:0.0:1767320783.068502:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320783.068504:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320783.068512:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9180 x1853163985512576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320783.068542:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320783.068545:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320783.068546:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320783.068549:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6026 00000100:00080000:0.0:1767320783.068550:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320783.068552:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320783.068560:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b800 x1853163985512832/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320783.068583:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320783.068585:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320783.068586:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320783.068588:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6026 00000100:00080000:0.0:1767320783.068590:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320783.068592:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320783.068597:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b100 x1853163985513088/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320783.068625:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985513216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320783.068676:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320783.068725:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320783.068728:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985512960:0@lo:400:kworker.0 00000100:00100000:0.0:1767320783.068746:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320783.068875:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985512704:0@lo:400:kworker.0 00000100:00100000:2.0:1767320783.068914:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320783.068940:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320783.068945:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985513344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320783.068965:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320783.068999:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513216 00000100:00100000:2.0:1767320783.069006:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513216 00000100:00100000:2.0:1767320783.069054:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513344 00000100:00100000:2.0:1767320783.069057:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513344 00000100:00100000:2.0:1767320783.069075:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3519 00000100:00100000:2.0:1767320783.069082:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985513216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320783.069118:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985513216:12345-0@lo:400:kworker.0 Request processed in 38us (442us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320783.069123:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3519 00000100:00100000:2.0:1767320783.069134:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3520 00000100:00100000:2.0:1767320783.069138:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985513344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320783.069154:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985513344:12345-0@lo:400:kworker.0 Request processed in 16us (189us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320783.069157:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3520 00000100:00080000:2.0:1767320783.069182:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320783.069185:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320783.069218:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985513344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320783.069244:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985513216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320783.069296:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512960 00000100:00100000:0.0:1767320783.069299:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512960 00000100:00100000:0.0:1767320783.069326:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512704 00000100:00100000:0.0:1767320783.069328:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512704 00000100:00100000:0.0:1767320783.069339:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5427 00000100:00100000:0.0:1767320783.069345:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985512960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320783.069377:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985512960:12345-0@lo:400:kworker.0 Request processed in 32us (629us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320783.069383:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5427 00000100:00100000:0.0:1767320783.069395:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5428 00000100:00100000:0.0:1767320783.069402:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985512704:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320783.069429:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985512704:12345-0@lo:400:kworker.0 Request processed in 28us (514us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320783.069434:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5428 00000100:00100000:3.0:1767320783.069509:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985512448:0@lo:38:kworker.0 00000100:00100000:3.0:1767320783.069530:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320783.069550:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985512576:0@lo:38:kworker.0 00000100:00100000:3.0:1767320783.069563:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320783.069574:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985512832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320783.069584:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320783.069594:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985513088:0@lo:38:kworker.0 00000100:00100000:3.0:1767320783.069604:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320783.069621:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512448 00000100:00100000:3.0:1767320783.069624:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512448 00000100:00100000:3.0:1767320783.069633:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512832 00000100:00100000:3.0:1767320783.069634:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512832 00000100:00100000:3.0:1767320783.069638:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513088 00000100:00100000:3.0:1767320783.069639:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513088 00000100:00100000:3.0:1767320783.069645:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6174 00000100:00100000:3.0:1767320783.069648:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985512448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320783.069661:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320783.069682:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985512448:12345-0@lo:38:kworker.0 Request processed in 33us (152us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320783.069686:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6174 00000100:00100000:3.0:1767320783.069693:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6175 00000100:00100000:3.0:1767320783.069696:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985512832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320783.069701:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320783.069712:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985512832:12345-0@lo:38:kworker.0 Request processed in 16us (127us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320783.069716:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6175 00000100:00100000:3.0:1767320783.069721:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6176 00000100:00100000:3.0:1767320783.069723:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985513088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320783.069728:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320783.069737:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985513088:12345-0@lo:38:kworker.0 Request processed in 14us (133us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320783.069741:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6176 00000100:00080000:3.0:1767320783.069764:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320783.069769:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320783.069772:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985512448:0@lo:38:kworker.0 00000100:00080000:3.0:1767320783.069779:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320783.069782:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320783.069784:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985512832:0@lo:38:kworker.0 00000100:00080000:3.0:1767320783.069790:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320783.069793:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320783.069796:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985513088:0@lo:38:kworker.0 00000100:00100000:2.0:1767320783.070013:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985512576 00000100:00100000:2.0:1767320783.070017:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985512576 00000100:00100000:2.0:1767320783.070037:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2034 00000100:00100000:2.0:1767320783.070040:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985512576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320783.070047:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320783.070073:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985512576:12345-0@lo:38:kworker.0 Request processed in 32us (509us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320783.070077:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2034 00000100:00080000:2.0:1767320783.070087:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320783.070099:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985512704:0@lo:400:kworker.0 00000100:00100000:2.0:1767320783.070111:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985512960:0@lo:400:kworker.0 00000100:00080000:0.0:1767320783.070153:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320783.070156:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320783.070158:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985512576:0@lo:38:kworker.0 00010000:00010000:1.0:1767320783.707524:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320784.604498:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320784.604517:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985513472:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320784.604581:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320784.604597:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320784.604610:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985513600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320784.604733:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320784.607383:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513472 00000100:00100000:0.0:1767320784.607399:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513472 00000100:00100000:0.0:1767320784.607463:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513600 00000100:00100000:0.0:1767320784.607467:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513600 00000100:00100000:3.0:1767320784.607479:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2730 00000100:00100000:3.0:1767320784.607490:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985513472:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320784.607506:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2731 00000100:00100000:0.0:1767320784.607518:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985513600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320784.607518:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:0.0:1767320784.607541:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320784.607585:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985513472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (3002us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320784.607591:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985513600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (2854us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320784.607597:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2730 00000100:00100000:0.0:1767320784.607605:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2731 00000100:00100000:3.0:1767320784.607659:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985513472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320784.608007:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985513600:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320784.731527:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320785.757592:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320786.779613:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320787.117290:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320787.117337:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320787.118200:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636544 00000100:00100000:2.0:1767320787.118212:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636544 00000100:00100000:2.0:1767320787.118261:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636672 00000100:00100000:2.0:1767320787.118265:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636672 00000100:00100000:2.0:1767320787.118329:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5429 00000100:00100000:2.0:1767320787.118340:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920636672:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320787.118411:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920636672:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (1072us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320787.118419:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5429 00000100:00100000:2.0:1767320787.118437:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6177 00000100:00100000:2.0:1767320787.118441:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920636544:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320787.118462:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320787.118487:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920636544:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (1219us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320787.118493:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6177 00000100:00080000:3.0:1767320787.119045:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320787.803451:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320788.187977:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320788.187987:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320788.187991:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320788.187995:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320788.187999:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320788.188007:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6031 00000100:00080000:2.0:1767320788.188012:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320788.188018:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320788.188054:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5595f80 x1853163985513728/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320788.188115:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985513728:0@lo:38:kworker.0 00000100:00100000:0.0:1767320788.188208:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320788.188358:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320788.188362:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320788.188364:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320788.188368:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6031 00000100:00080000:2.0:1767320788.188370:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320788.188374:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320788.188392:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5597100 x1853163985513856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320788.189057:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985513856:0@lo:38:kworker.0 00000100:00100000:0.0:1767320788.189100:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320788.189335:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513728 00000100:00100000:1.0:1767320788.189343:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513728 00000100:00100000:1.0:1767320788.189383:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6178 00000100:00100000:1.0:1767320788.189390:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985513728:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320788.189411:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320788.189465:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985513728:12345-0@lo:38:kworker.0 Request processed in 73us (1267us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320788.189471:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6178 00000100:00080000:0.0:1767320788.190693:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320788.190703:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320788.190708:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985513728:0@lo:38:kworker.0 00000100:00080000:2.0:1767320788.191132:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320788.191137:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320788.191139:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320788.191145:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6031 00000100:00080000:2.0:1767320788.191149:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320788.191154:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320788.191189:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5596680 x1853163985514112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320788.191336:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985513984:0@lo:400:kworker.0 00000100:00100000:1.0:1767320788.191395:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320788.191441:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513984 00000100:00100000:1.0:1767320788.191447:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513984 00000100:00100000:1.0:1767320788.191494:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5430 00000100:00100000:1.0:1767320788.191502:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985513984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320788.191552:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985513984:12345-0@lo:400:kworker.0 Request processed in 51us (157us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320788.191559:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5430 00000100:00080000:1.0:1767320788.191590:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320788.191612:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985513984:0@lo:400:kworker.0 00000100:00100000:1.0:1767320788.191798:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320788.191837:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985514240:0@lo:400:kworker.0 00000100:00100000:1.0:1767320788.191862:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320788.191893:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985514240 00000100:00100000:1.0:1767320788.191897:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985514240 00000100:00100000:1.0:1767320788.191925:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5431 00000100:00100000:1.0:1767320788.191931:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985514240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320788.191963:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985514240:12345-0@lo:400:kworker.0 Request processed in 33us (100us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320788.191969:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5431 00000100:00080000:1.0:1767320788.191982:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320788.191998:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985514240:0@lo:400:kworker.0 00000100:00080000:2.0:1767320788.192061:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320788.192066:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320788.192069:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320788.192073:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6031 00000100:00080000:2.0:1767320788.192076:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320788.192079:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320788.192097:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5594380 x1853163985514368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320788.192152:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985514624:0@lo:400:kworker.0 00000100:00100000:2.0:1767320788.192203:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320788.192324:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985514624 00000100:00100000:2.0:1767320788.192329:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985514624 00000100:00100000:2.0:1767320788.192359:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3521 00000100:00100000:2.0:1767320788.192365:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985514624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320788.192396:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985514624:12345-0@lo:400:kworker.0 Request processed in 31us (192us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320788.192401:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3521 00000100:00100000:2.0:1767320788.192429:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985514624:0@lo:400:kworker.0 00000100:00080000:2.0:1767320788.192442:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320788.192497:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320788.192505:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985514496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320788.192970:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320788.195217:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985514112:0@lo:38:kworker.0 00000100:00100000:0.0:1767320788.195450:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320788.195495:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985514368:0@lo:38:kworker.0 00000100:00100000:0.0:1767320788.195519:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320788.195566:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985514112 00000100:00100000:0.0:1767320788.195590:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985514112 00000100:00100000:0.0:1767320788.195621:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985514368 00000100:00100000:0.0:1767320788.195625:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985514368 00000100:00100000:0.0:1767320788.195643:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6179 00000100:00100000:0.0:1767320788.195650:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985514112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320788.195671:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320788.195714:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985514112:12345-0@lo:38:kworker.0 Request processed in 63us (262us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320788.195721:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6179 00000100:00100000:0.0:1767320788.195737:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6180 00000100:00100000:0.0:1767320788.195741:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985514368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320788.195751:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320788.195773:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985514368:12345-0@lo:38:kworker.0 Request processed in 30us (252us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320788.195778:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6180 00000100:00080000:0.0:1767320788.195823:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320788.195828:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320788.195832:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985514112:0@lo:38:kworker.0 00000100:00080000:0.0:1767320788.195841:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320788.195845:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320788.195847:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985514368:0@lo:38:kworker.0 00000100:00100000:2.0:1767320788.196202:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985514496 00000100:00100000:2.0:1767320788.196208:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985514496 00000100:00100000:2.0:1767320788.196263:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3522 00000100:00100000:2.0:1767320788.196271:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985514496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320788.196317:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985514496:12345-0@lo:400:kworker.0 Request processed in 48us (3346us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320788.196324:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3522 00000100:00100000:1.0:1767320788.196588:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985514496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320788.196892:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985513856 00000100:00100000:3.0:1767320788.196898:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985513856 00000100:00100000:3.0:1767320788.196931:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2035 00000100:00100000:3.0:1767320788.196936:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985513856:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320788.196950:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320788.196994:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985513856:12345-0@lo:38:kworker.0 Request processed in 57us (7893us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320788.197000:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2035 00000100:00080000:0.0:1767320788.197065:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320788.197124:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320788.197129:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320788.197132:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985513856:0@lo:38:kworker.0 00010000:00010000:1.0:1767320788.829526:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320789.727197:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320789.727223:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985514752:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320789.727294:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320789.727429:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985514752 00000100:00100000:2.0:1767320789.727438:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985514752 00000100:00100000:2.0:1767320789.727500:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2732 00000100:00100000:2.0:1767320789.727509:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985514752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320789.727539:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320789.727590:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985514752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320789.727597:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2732 00000100:00100000:0.0:1767320789.727745:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985514752:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320789.728260:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985514880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320789.728295:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320789.728518:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985514880 00000100:00100000:3.0:1767320789.728525:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985514880 00000100:00100000:3.0:1767320789.728596:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2733 00000100:00100000:3.0:1767320789.728605:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985514880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320789.728626:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320789.729044:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985514880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 439us (746us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320789.729051:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2733 00000100:00100000:2.0:1767320789.729195:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985514880:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320789.852967:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320790.875390:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320791.900932:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320792.234096:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767320792.234136:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920637056 offset 0 length 520: 4 00000100:00100000:2.0:1767320792.234196:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320792.234280:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636800 00000100:00100000:1.0:1767320792.234300:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636800 00000100:00100000:1.0:1767320792.234341:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920636928 00000100:00100000:1.0:1767320792.234344:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920636928 00000100:00100000:1.0:1767320792.234407:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5432 00000100:00100000:1.0:1767320792.234418:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920636928:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320792.234491:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920636928:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (328us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320792.234499:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5432 00000100:00100000:1.0:1767320792.234516:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6181 00000100:00100000:1.0:1767320792.234521:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920636800:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320792.234540:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320792.234577:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920636800:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (495us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320792.234583:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6181 00000100:00080000:0.0:1767320792.235253:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320792.284390:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320767/real 1767320767] req@ffff8ce1b3579880 x1853163985508480/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320792 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320792.284424:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3579880 x1853163985508480/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320792 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320792.284445:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320792.284456:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320792.284460:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985508480:0@lo:250:kworker.0 00000100:00080000:0.0:1767320792.285333:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320792.285340:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320792.285342:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320792.285348:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6015 00000100:00080000:0.0:1767320792.285352:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320792.285359:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320792.285387:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985515008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320792.285415:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320792.285419:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320792.285421:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320792.285425:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6036 00000100:00080000:0.0:1767320792.285427:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320792.285431:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320792.285446:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffd180 x1853163985515136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320792.285456:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320792.285459:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320792.285460:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320792.285463:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6036 00000100:00080000:0.0:1767320792.285466:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320792.285469:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320792.285479:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffca80 x1853163985515264/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320792.285510:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320792.285513:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320792.285515:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320792.285518:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6036 00000100:00080000:0.0:1767320792.285521:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320792.285524:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320792.285535:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffe680 x1853163985515520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320792.285560:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320792.285563:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320792.285565:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320792.285569:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6036 00000100:00080000:0.0:1767320792.285571:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320792.285574:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320792.285587:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff480 x1853163985515776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320792.285637:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985515392:0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.285700:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.285771:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320792.285777:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985516032:0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.285804:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.285838:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985515648:0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.285859:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.285876:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320792.286012:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985515904:0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.286039:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.286071:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516032 00000100:00100000:0.0:1767320792.286198:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516032 00000100:00100000:0.0:1767320792.286251:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985515904 00000100:00100000:0.0:1767320792.286255:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985515904 00000100:00100000:0.0:1767320792.286277:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3523 00000100:00100000:0.0:1767320792.286286:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985516032:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.286336:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985516032:12345-0@lo:400:kworker.0 Request processed in 51us (531us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320792.286343:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3523 00000100:00100000:0.0:1767320792.286360:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3524 00000100:00100000:0.0:1767320792.286366:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985515904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.286395:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985515904:12345-0@lo:400:kworker.0 Request processed in 29us (355us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320792.286401:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3524 00000100:00100000:0.0:1767320792.286417:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985515392 00000100:00100000:0.0:1767320792.286420:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985515392 00000100:00100000:0.0:1767320792.286445:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985515648 00000100:00100000:0.0:1767320792.286448:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985515648 00000100:00100000:0.0:1767320792.286465:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5433 00000100:00100000:0.0:1767320792.286470:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985515392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.286498:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985515392:12345-0@lo:400:kworker.0 Request processed in 28us (798us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320792.286504:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5433 00000100:00100000:0.0:1767320792.286515:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5434 00000100:00100000:0.0:1767320792.286521:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985515648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.286547:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985515648:12345-0@lo:400:kworker.0 Request processed in 26us (686us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320792.286552:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5434 00000100:00080000:0.0:1767320792.286571:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320792.286575:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320792.286577:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320792.286613:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985515904:0@lo:400:kworker.0 00000100:00100000:2.0:1767320792.286745:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985515008:0@lo:250:kworker.0 00000100:00100000:0.0:1767320792.287571:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985515648:0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.293490:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985516032:0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.293520:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985515392:0@lo:400:kworker.0 00000400:00000100:2.0:1767320792.293535:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985515008 offset 0 length 520: 4 00000100:00100000:2.0:1767320792.293595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985515136:0@lo:38:kworker.0 00000100:00100000:2.0:1767320792.294300:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320792.294370:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985515264:0@lo:38:kworker.0 00000100:00100000:2.0:1767320792.294396:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320792.294418:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985515520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320792.294436:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320792.294507:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985515264 00000100:00100000:3.0:1767320792.294519:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985515264 00000100:00100000:3.0:1767320792.294634:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2036 00000100:00100000:3.0:1767320792.294644:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985515264:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320792.294672:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320792.294756:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985515264:12345-0@lo:38:kworker.0 Request processed in 111us (356us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320792.294764:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2036 00000100:00100000:2.0:1767320792.297368:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985515776:0@lo:38:kworker.0 00000100:00100000:2.0:1767320792.297429:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320792.301332:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985515136 00000100:00100000:1.0:1767320792.301349:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985515136 00000100:00100000:1.0:1767320792.301416:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985515520 00000100:00100000:1.0:1767320792.301420:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985515520 00000100:00100000:1.0:1767320792.301469:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6182 00000100:00100000:1.0:1767320792.301481:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985515136:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320792.301490:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6183 00000100:00100000:0.0:1767320792.301504:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985515520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320792.301516:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767320792.301528:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320792.304290:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985515520:12345-0@lo:38:kworker.0 Request processed in 2771us (9835us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320792.304309:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6183 00000100:00100000:0.0:1767320792.304353:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985515776 00000100:00100000:0.0:1767320792.304363:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985515776 00000100:00100000:0.0:1767320792.304427:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6184 00000100:00100000:0.0:1767320792.304434:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985515776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320792.304460:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320792.304526:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985515776:12345-0@lo:38:kworker.0 Request processed in 93us (7098us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320792.304532:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6184 00000100:00100000:1.0:1767320792.306560:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985515136:12345-0@lo:38:kworker.0 Request processed in 5069us (12249us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320792.306577:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6182 00000100:00080000:2.0:1767320792.306924:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320792.306931:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320792.306935:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985515776:0@lo:38:kworker.0 00000100:00080000:2.0:1767320792.306968:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320792.306972:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320792.306975:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985515136:0@lo:38:kworker.0 00000100:00080000:2.0:1767320792.306986:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320792.306990:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320792.306993:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985515264:0@lo:38:kworker.0 00000100:00080000:2.0:1767320792.307003:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320792.307006:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320792.307009:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985515520:0@lo:38:kworker.0 00010000:00010000:0.0:1767320792.923328:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320793.947404:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320794.843648:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320794.843667:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985516160:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320794.843760:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320794.843955:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516160 00000100:00100000:1.0:1767320794.843963:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516160 00000100:00100000:1.0:1767320794.844028:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2734 00000100:00100000:1.0:1767320794.844038:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985516160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320794.844066:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320794.844126:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985516160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (358us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320794.844134:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2734 00000100:00080000:2.0:1767320794.844822:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320794.844865:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985516160:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320794.845193:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320794.845206:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985516288:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320794.845247:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320794.845321:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516288 00000100:00100000:1.0:1767320794.845328:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516288 00000100:00100000:1.0:1767320794.845783:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2735 00000100:00100000:1.0:1767320794.845793:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985516288:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320794.845817:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320794.845871:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985516288:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (622us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320794.845878:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2735 00000100:00080000:2.0:1767320794.845936:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320794.846037:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985516288:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320794.972467:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320795.995503:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320797.019992:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320797.355580:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320797.355724:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920637312 00000100:00100000:0.0:1767320797.355738:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920637312 00000100:00100000:0.0:1767320797.355813:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5435 00000100:00100000:0.0:1767320797.355822:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920637312:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320797.355897:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920637312:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (342us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320797.355902:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5435 00000100:00080000:2.0:1767320797.355984:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320797.358821:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320797.358933:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920637184 00000100:00100000:3.0:1767320797.358947:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920637184 00000100:00100000:3.0:1767320797.359018:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6185 00000100:00100000:3.0:1767320797.359029:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920637184:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320797.359062:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320797.359207:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920637184:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 134us (356us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320797.359218:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6185 00000100:00080000:3.0:1767320797.403846:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320797.403858:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320797.403861:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320797.403866:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320797.403870:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320797.403879:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6040 00000100:00080000:3.0:1767320797.403884:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320797.403891:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320797.403950:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c700 x1853163985516416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320797.403978:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320797.403981:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320797.403982:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320797.403985:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6040 00000100:00080000:3.0:1767320797.403987:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320797.403989:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320797.404001:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89f80 x1853163985516544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320797.404048:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767320797.404053:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985516416:0@lo:38:kworker.0 00000100:00080000:3.0:1767320797.404053:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320797.404056:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320797.404068:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6040 00000100:00080000:3.0:1767320797.404076:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320797.404081:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320797.404096:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b800 x1853163985516800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320797.404135:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320797.404140:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320797.404143:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320797.404147:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6040 00000100:00080000:3.0:1767320797.404152:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320797.404205:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320797.404279:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767320797.404993:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320797.405025:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb100 x1853163985517056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320797.405078:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320797.405094:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985517184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.405182:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320797.405258:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320797.405265:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985517312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.405285:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320797.405314:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517184 00000100:00100000:2.0:1767320797.405365:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.405428:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516416 00000100:00100000:2.0:1767320797.405446:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516416 00000100:00100000:2.0:1767320797.405497:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6186 00000100:00100000:2.0:1767320797.405505:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985516416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320797.405526:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320797.405587:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985516416:12345-0@lo:38:kworker.0 Request processed in 80us (1434us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320797.405593:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6186 00000100:00080000:2.0:1767320797.405656:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.405661:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.405665:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985516416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.405674:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985516544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.405710:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.405734:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516544 00000100:00100000:2.0:1767320797.405737:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516544 00000100:00100000:2.0:1767320797.405759:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2037 00000100:00100000:2.0:1767320797.405764:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985516544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320797.405773:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320797.405797:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985516544:12345-0@lo:38:kworker.0 Request processed in 32us (87us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320797.405807:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2037 00000100:00080000:2.0:1767320797.405837:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.405841:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.405844:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985516544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.405871:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985516800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.405893:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.405911:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516800 00000100:00100000:2.0:1767320797.405915:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516800 00000100:00100000:2.0:1767320797.405933:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6187 00000100:00100000:2.0:1767320797.405937:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985516800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320797.405946:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320797.405970:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985516800:12345-0@lo:38:kworker.0 Request processed in 32us (76us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320797.405976:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6187 00000100:00080000:2.0:1767320797.405998:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.406002:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.406005:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985516800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.406011:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985517056:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.406031:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.406047:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517056 00000100:00100000:2.0:1767320797.406050:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517056 00000100:00100000:2.0:1767320797.406066:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6188 00000100:00100000:2.0:1767320797.406070:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985517056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320797.406080:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320797.406102:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985517056:12345-0@lo:38:kworker.0 Request processed in 30us (71us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320797.406107:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6188 00000100:00080000:2.0:1767320797.406130:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.406134:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.406137:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985517056:0@lo:38:kworker.0 00000100:00100000:0.0:1767320797.406630:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985516928:0@lo:400:kworker.0 00000100:00100000:0.0:1767320797.406672:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320797.406758:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517312 00000100:00100000:0.0:1767320797.406762:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517312 00000100:00100000:0.0:1767320797.406805:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3525 00000100:00100000:0.0:1767320797.406812:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985517312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320797.406857:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985517312:12345-0@lo:400:kworker.0 Request processed in 45us (1569us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320797.406864:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3525 00000100:00100000:3.0:1767320797.406882:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517184 00000100:00080000:0.0:1767320797.406894:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320797.406961:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985517312:0@lo:400:kworker.0 00000100:00100000:0.0:1767320797.406973:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3526 00000100:00100000:0.0:1767320797.406983:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985517184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320797.407036:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985517184:12345-0@lo:400:kworker.0 Request processed in 55us (1873us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320797.407041:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3526 00000100:00080000:0.0:1767320797.407060:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320797.407240:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985517184:0@lo:400:kworker.0 00000100:00100000:1.0:1767320797.409496:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985516672:0@lo:400:kworker.0 00000100:00100000:1.0:1767320797.409604:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.414564:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516928 00000100:00100000:2.0:1767320797.414728:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516928 00000100:00100000:2.0:1767320797.414898:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985516672 00000100:00100000:2.0:1767320797.414903:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985516672 00000100:00080000:0.0:1767320797.414935:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320797.414975:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5436 00000100:00100000:0.0:1767320797.414987:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985516928:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320797.415145:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5437 00000100:00100000:2.0:1767320797.415181:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985516672:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320797.418204:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985516928:12345-0@lo:400:kworker.0 Request processed in 3209us (11520us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320797.418218:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5436 00000100:00100000:3.0:1767320797.418429:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985516928:0@lo:400:kworker.0 00000100:00100000:2.0:1767320797.421136:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985516672:12345-0@lo:400:kworker.0 Request processed in 5977us (11524us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320797.421148:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5437 00000100:00100000:1.0:1767320797.424657:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985516672:0@lo:400:kworker.0 00010000:00010000:2.0:1767320798.045206:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320799.068270:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320799.963509:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985517440:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320799.963608:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320799.963727:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517440 00000100:00100000:1.0:1767320799.963747:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517440 00000100:00100000:1.0:1767320799.963849:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2736 00000100:00100000:1.0:1767320799.963862:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985517440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320799.963906:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767320799.964012:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320799.965920:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320799.965956:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985517568:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320799.966033:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320799.966103:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517568 00000100:00100000:3.0:1767320799.966111:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517568 00000100:00100000:3.0:1767320799.966200:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2737 00000100:00100000:3.0:1767320799.966210:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985517568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320799.966248:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320799.966294:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985517568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (260us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320799.966304:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2737 00000100:00080000:3.0:1767320799.966329:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320799.966369:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985517568:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320799.967568:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985517440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3699us (3978us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320799.967582:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2736 00000100:00100000:0.0:1767320799.970106:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985517440:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320800.091593:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320801.115750:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320802.140149:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320802.477531:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320802.477761:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320802.479126:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920637568 00000100:00100000:2.0:1767320802.479146:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920637568 00000100:00080000:3.0:1767320802.479345:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320802.479441:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920637440 00000100:00100000:2.0:1767320802.479447:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920637440 00000100:00100000:2.0:1767320802.479526:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5438 00000100:00100000:2.0:1767320802.479558:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920637568:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320802.479691:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920637568:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 154us (2176us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320802.479701:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5438 00000100:00100000:2.0:1767320802.479725:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6189 00000100:00100000:2.0:1767320802.479730:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920637440:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320802.479757:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320802.479793:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920637440:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 62us (2033us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320802.479799:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6189 00000100:00080000:1.0:1767320802.524432:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320802.524444:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320802.524447:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320802.524451:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320802.524456:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320802.524466:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6045 00000100:00080000:1.0:1767320802.524471:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320802.524478:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320802.524528:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985517696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320802.524602:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985517696:0@lo:38:kworker.0 00000100:00100000:2.0:1767320802.524686:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320802.525729:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517696 00000100:00100000:0.0:1767320802.525742:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517696 00000100:00100000:0.0:1767320802.525806:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6190 00000100:00100000:0.0:1767320802.525815:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985517696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320802.525845:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320802.525963:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320802.525969:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320802.525972:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320802.525977:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6045 00000100:00080000:1.0:1767320802.525980:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320802.525985:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320802.526012:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985517824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320802.527532:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985517824:0@lo:38:kworker.0 00000100:00100000:2.0:1767320802.527623:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320802.527818:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517824 00000100:00100000:3.0:1767320802.527827:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517824 00000100:00100000:3.0:1767320802.527869:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2038 00000100:00100000:3.0:1767320802.527877:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985517824:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320802.527901:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320802.528076:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985517696:12345-0@lo:38:kworker.0 Request processed in 2259us (3387us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320802.528088:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6190 00000100:00100000:3.0:1767320802.528205:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985517824:12345-0@lo:38:kworker.0 Request processed in 328us (582us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320802.528221:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2038 00000100:00080000:2.0:1767320802.528369:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320802.528375:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320802.528379:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985517696:0@lo:38:kworker.0 00000100:00080000:2.0:1767320802.528407:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320802.528410:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320802.528413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985517824:0@lo:38:kworker.0 00000100:00100000:3.0:1767320802.528625:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985517952:0@lo:400:kworker.0 00000100:00100000:3.0:1767320802.528705:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320802.528780:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320802.528786:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320802.528789:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320802.528794:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6045 00000100:00080000:1.0:1767320802.528797:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320802.528803:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320802.528827:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985518080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320802.528860:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320802.528863:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320802.528866:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320802.528870:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6045 00000100:00080000:1.0:1767320802.528872:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320802.528876:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320802.528885:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9500 x1853163985518336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320802.528948:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985518592:0@lo:400:kworker.0 00000100:00100000:2.0:1767320802.528979:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320802.529014:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518592 00000100:00100000:2.0:1767320802.529019:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518592 00000100:00100000:2.0:1767320802.529065:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3527 00000100:00100000:2.0:1767320802.529073:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985518592:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320802.529113:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985518592:12345-0@lo:400:kworker.0 Request processed in 42us (134us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320802.529120:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3527 00000100:00100000:2.0:1767320802.529147:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985518592:0@lo:400:kworker.0 00000100:00080000:2.0:1767320802.529193:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320802.529378:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985517952 00000100:00100000:0.0:1767320802.529383:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985517952 00000100:00100000:0.0:1767320802.529413:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5439 00000100:00100000:0.0:1767320802.529419:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985517952:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320802.529474:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320802.529640:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985518208:0@lo:400:kworker.0 00000100:00100000:0.0:1767320802.529709:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985517952:12345-0@lo:400:kworker.0 Request processed in 289us (1003us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320802.529716:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5439 00000100:00100000:0.0:1767320802.529738:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985518464:0@lo:400:kworker.0 00000100:00100000:0.0:1767320802.529765:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320802.529807:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985518080:0@lo:38:kworker.0 00000100:00100000:0.0:1767320802.529830:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320802.529842:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985518336:0@lo:38:kworker.0 00000100:00100000:0.0:1767320802.529857:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320802.529876:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518080 00000100:00100000:0.0:1767320802.529879:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518080 00000100:00100000:0.0:1767320802.529893:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518336 00000100:00100000:0.0:1767320802.529895:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518336 00000100:00100000:0.0:1767320802.529914:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6191 00000100:00100000:0.0:1767320802.529930:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985518080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320802.529948:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320802.529977:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985518080:12345-0@lo:38:kworker.0 Request processed in 47us (146us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320802.529983:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6191 00000100:00100000:0.0:1767320802.529995:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6192 00000100:00100000:0.0:1767320802.529999:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985518336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320802.530008:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320802.530024:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985518336:12345-0@lo:38:kworker.0 Request processed in 24us (166us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320802.530029:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6192 00000100:00080000:0.0:1767320802.530055:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320802.530059:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320802.530063:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985518080:0@lo:38:kworker.0 00000100:00080000:0.0:1767320802.530069:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320802.530073:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320802.530075:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985518336:0@lo:38:kworker.0 00000100:00100000:2.0:1767320802.530234:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518464 00000100:00100000:2.0:1767320802.530238:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518464 00000100:00100000:2.0:1767320802.530267:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3528 00000100:00100000:2.0:1767320802.530273:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985518464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320802.530310:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985518464:12345-0@lo:400:kworker.0 Request processed in 38us (545us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320802.530316:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3528 00000100:00080000:2.0:1767320802.530329:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320802.530347:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985518464:0@lo:400:kworker.0 00000100:00100000:1.0:1767320802.530501:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320802.530899:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518208 00000100:00100000:2.0:1767320802.530904:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518208 00000100:00100000:2.0:1767320802.530953:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5440 00000100:00100000:2.0:1767320802.530961:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985518208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320802.531405:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985517952:0@lo:400:kworker.0 00000100:00080000:3.0:1767320802.531446:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320802.531610:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985518208:12345-0@lo:400:kworker.0 Request processed in 649us (1107us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320802.531619:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5440 00000100:00100000:1.0:1767320802.531747:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985518208:0@lo:400:kworker.0 00010000:00010000:1.0:1767320803.164769:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320804.190444:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320805.083453:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985518848:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320805.083541:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320805.083560:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985518720:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320805.087813:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320805.088066:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518848 00000100:00100000:1.0:1767320805.088080:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518848 00000100:00100000:1.0:1767320805.088197:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2738 00000100:00100000:1.0:1767320805.088210:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985518848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320805.088259:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320805.088329:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985518848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 123us (527us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320805.088338:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2738 00000100:00100000:0.0:1767320805.088452:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985518848:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320805.088532:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320805.088866:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320805.089008:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518720 00000100:00100000:0.0:1767320805.089016:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518720 00000100:00100000:0.0:1767320805.089070:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2739 00000100:00100000:0.0:1767320805.089079:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985518720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320805.089101:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320805.089147:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985518720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320805.089155:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2739 00000100:00080000:3.0:1767320805.089232:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320805.089798:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985518720:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320805.211484:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320806.240464:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320807.260352:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320807.596920:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320807.597360:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320807.598033:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920637824 00000100:00100000:2.0:1767320807.598047:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920637824 00000100:00100000:2.0:1767320807.598138:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920637696 00000100:00100000:2.0:1767320807.598142:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920637696 00000100:00100000:2.0:1767320807.598184:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5441 00000100:00100000:2.0:1767320807.598198:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920637824:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320807.598299:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920637824:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 106us (1392us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320807.598307:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5441 00000100:00080000:0.0:1767320807.598778:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320807.598850:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6193 00000100:00100000:2.0:1767320807.598858:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920637696:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320807.598882:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320807.598917:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920637696:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 60us (1560us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320807.599534:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6193 00000100:00080000:0.0:1767320807.643796:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320807.643808:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320807.643812:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320807.643817:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320807.643822:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320807.643833:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6050 00000100:00080000:0.0:1767320807.643840:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320807.643850:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320807.643907:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75f80 x1853163985518976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320807.643972:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985518976:0@lo:38:kworker.0 00000100:00100000:1.0:1767320807.644063:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320807.644272:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985518976 00000100:00080000:0.0:1767320807.644427:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320807.644431:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320807.644434:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320807.644439:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6050 00000100:00080000:0.0:1767320807.644442:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320807.644457:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320807.644476:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163985519104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320807.644504:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985518976 00000100:00100000:2.0:1767320807.644568:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6194 00000100:00100000:2.0:1767320807.644577:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985518976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320807.644605:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320807.645618:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985519104:0@lo:38:kworker.0 00000100:00100000:1.0:1767320807.645678:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320807.645894:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320807.645899:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320807.645902:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320807.645907:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6050 00000100:00080000:0.0:1767320807.645911:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320807.645916:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320807.645935:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77100 x1853163985519360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320807.645967:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320807.645971:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320807.645973:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320807.645977:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6050 00000100:00080000:0.0:1767320807.645979:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320807.645983:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320807.645994:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75500 x1853163985519616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320807.646037:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985519872:0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646074:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320807.646100:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320807.646106:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985519232:0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646133:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320807.646194:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985519744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646219:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320807.646240:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320807.646244:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985519488:0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646262:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320807.646289:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985519232 00000100:00100000:0.0:1767320807.646295:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985519232 00000100:00100000:0.0:1767320807.646348:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985519488 00000100:00100000:0.0:1767320807.646350:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985519488 00000100:00100000:0.0:1767320807.646373:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5442 00000100:00100000:0.0:1767320807.646381:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985519232:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646461:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985519232:12345-0@lo:400:kworker.0 Request processed in 80us (325us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320807.646470:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5442 00000100:00100000:0.0:1767320807.646489:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5443 00000100:00100000:0.0:1767320807.646495:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985519488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646527:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985519488:12345-0@lo:400:kworker.0 Request processed in 32us (263us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320807.646533:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5443 00000100:00080000:0.0:1767320807.646558:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320807.646599:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985519488:0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646629:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985519232:0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646656:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985519872 00000100:00100000:0.0:1767320807.646659:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985519872 00000100:00100000:0.0:1767320807.646688:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985519744 00000100:00100000:0.0:1767320807.646690:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985519744 00000100:00100000:0.0:1767320807.646708:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3529 00000100:00100000:0.0:1767320807.646713:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985519872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.646744:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985519872:12345-0@lo:400:kworker.0 Request processed in 30us (670us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320807.646749:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3529 00000100:00100000:0.0:1767320807.646759:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3530 00000100:00100000:0.0:1767320807.646921:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985519744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.647042:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985519744:12345-0@lo:400:kworker.0 Request processed in 144us (822us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320807.647050:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3530 00000100:00080000:0.0:1767320807.647101:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320807.647106:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320807.647153:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985519744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320807.647224:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985519872:0@lo:400:kworker.0 00000100:00100000:3.0:1767320807.648547:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985519104 00000100:00100000:3.0:1767320807.648561:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985519104 00000100:00100000:3.0:1767320807.648626:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2039 00000100:00100000:3.0:1767320807.648635:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985519104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320807.648662:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320807.648738:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985519104:12345-0@lo:38:kworker.0 Request processed in 103us (3059us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320807.648750:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2039 00000100:00080000:1.0:1767320807.649598:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320807.649607:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320807.649613:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985518976:0@lo:38:kworker.0 00000100:00100000:1.0:1767320807.649651:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985519360:0@lo:38:kworker.0 00000100:00100000:2.0:1767320807.649818:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985518976:12345-0@lo:38:kworker.0 Request processed in 5238us (5751us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320807.649829:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6194 00000100:00100000:1.0:1767320807.649843:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320807.649924:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985519616:0@lo:38:kworker.0 00000100:00100000:1.0:1767320807.649947:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320807.656043:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985519360 00000100:00100000:2.0:1767320807.656065:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985519360 00000100:00100000:2.0:1767320807.656157:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6195 00000100:00100000:2.0:1767320807.656178:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985519360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320807.656208:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320807.656327:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985519616 00000100:00100000:3.0:1767320807.656333:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985519616 00000100:00100000:3.0:1767320807.656362:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6196 00000100:00100000:3.0:1767320807.656368:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985519616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320807.656379:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320807.656457:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320807.656464:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320807.656469:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985519104:0@lo:38:kworker.0 00000100:00080000:1.0:1767320807.656504:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320807.656508:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320807.656511:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985519360:0@lo:38:kworker.0 00000100:00100000:2.0:1767320807.657319:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985519360:12345-0@lo:38:kworker.0 Request processed in 1138us (7471us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320807.657330:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6195 00000100:00100000:3.0:1767320807.657552:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985519616:12345-0@lo:38:kworker.0 Request processed in 1182us (7603us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320807.657560:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6196 00000100:00080000:1.0:1767320807.657619:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320807.657624:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320807.657628:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985519616:0@lo:38:kworker.0 00010000:00010000:1.0:1767320808.289354:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320809.311210:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320810.205590:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320810.205625:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985520000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320810.205741:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320810.205950:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520000 00000100:00100000:2.0:1767320810.205971:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520000 00000100:00080000:3.0:1767320810.206197:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320810.206309:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2740 00000100:00100000:2.0:1767320810.206323:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985520000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320810.206487:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320810.206549:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985520000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 231us (808us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320810.206558:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2740 00000100:00100000:1.0:1767320810.206739:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985520000:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320810.206803:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320810.206818:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985520128:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320810.206858:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320810.207346:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520128 00000100:00100000:2.0:1767320810.207354:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520128 00000100:00100000:2.0:1767320810.207437:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2741 00000100:00100000:2.0:1767320810.207447:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985520128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320810.207469:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767320810.207497:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320810.207522:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985520128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (663us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320810.207530:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2741 00000100:00100000:1.0:1767320810.208143:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985520128:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320810.331719:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320811.355621:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320812.379661:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320812.714898:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320812.715095:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920637952 00000100:00100000:3.0:1767320812.715107:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920637952 00000100:00100000:3.0:1767320812.715239:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6197 00000100:00100000:3.0:1767320812.715250:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920637952:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320812.715371:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320812.715518:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920637952:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 266us (633us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320812.715528:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6197 00000100:00100000:3.0:1767320812.718024:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320812.718204:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920638080 00000100:00100000:0.0:1767320812.718216:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920638080 00000100:00080000:3.0:1767320812.718275:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320812.718303:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5444 00000100:00100000:0.0:1767320812.718315:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920638080:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320812.718432:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920638080:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 119us (423us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320812.718441:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5444 00000100:00080000:2.0:1767320812.763815:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320812.763830:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320812.763835:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320812.763839:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320812.763845:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320812.763857:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6055 00000100:00080000:2.0:1767320812.763864:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320812.763874:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320812.763932:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578380 x1853163985520256/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320812.763962:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320812.763965:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320812.763966:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320812.763969:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6055 00000100:00080000:2.0:1767320812.763971:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320812.763976:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320812.763992:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9180 x1853163985520384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320812.764048:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320812.764051:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320812.764053:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320812.764056:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6055 00000100:00080000:2.0:1767320812.764058:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320812.764061:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320812.764071:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb100 x1853163985520640/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320812.764097:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320812.764099:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320812.764101:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320812.764103:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6055 00000100:00080000:2.0:1767320812.764106:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320812.764108:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320812.764115:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b800 x1853163985520896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320812.764158:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985521024:0@lo:400:kworker.0 00000100:00100000:2.0:1767320812.764255:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320812.764319:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320812.764323:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985520768:0@lo:400:kworker.0 00000100:00100000:2.0:1767320812.764342:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320812.764394:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985521024 00000100:00100000:2.0:1767320812.764404:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985521024 00000100:00100000:1.0:1767320812.764458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985520256:0@lo:38:kworker.0 00000100:00100000:2.0:1767320812.764477:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3531 00000100:00100000:2.0:1767320812.764487:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985521024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.764494:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320812.764519:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985520384:0@lo:38:kworker.0 00000100:00100000:2.0:1767320812.764543:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985521024:12345-0@lo:400:kworker.0 Request processed in 57us (287us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320812.764551:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320812.764552:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3531 00000100:00100000:1.0:1767320812.764574:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985520640:0@lo:38:kworker.0 00000100:00100000:1.0:1767320812.764595:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320812.764603:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985521024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.764608:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985520896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320812.764627:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320812.764699:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985520512:0@lo:400:kworker.0 00000100:00100000:3.0:1767320812.764883:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985521152:0@lo:400:kworker.0 00000100:00100000:3.0:1767320812.764916:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320812.764951:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320812.764963:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985521152 00000100:00100000:3.0:1767320812.764967:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985521152 00000100:00100000:3.0:1767320812.764998:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3532 00000100:00100000:3.0:1767320812.765004:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985521152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320812.765040:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985521152:12345-0@lo:400:kworker.0 Request processed in 35us (122us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320812.765045:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3532 00000100:00080000:3.0:1767320812.765061:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320812.765077:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985521152:0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.765184:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520384 00000100:00100000:1.0:1767320812.765189:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520384 00000100:00100000:1.0:1767320812.765215:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2040 00000100:00100000:1.0:1767320812.765220:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985520384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320812.765243:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320812.765279:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985520384:12345-0@lo:38:kworker.0 Request processed in 58us (739us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320812.765285:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2040 00000100:00100000:2.0:1767320812.765383:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520768 00000100:00100000:2.0:1767320812.765388:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520768 00000100:00100000:2.0:1767320812.765429:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5445 00000100:00100000:2.0:1767320812.765436:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985520768:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320812.765475:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985520768:12345-0@lo:400:kworker.0 Request processed in 39us (1131us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320812.765481:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5445 00000100:00100000:2.0:1767320812.765492:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520256 00000100:00100000:2.0:1767320812.765496:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520256 00000100:00100000:2.0:1767320812.765512:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520640 00000100:00100000:2.0:1767320812.765516:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520640 00000100:00100000:2.0:1767320812.765524:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520896 00000100:00100000:2.0:1767320812.765527:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520896 00000100:00100000:2.0:1767320812.765538:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6198 00000100:00100000:2.0:1767320812.765542:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985520256:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320812.765553:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320812.765576:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985520256:12345-0@lo:38:kworker.0 Request processed in 32us (1081us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320812.765582:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6198 00000100:00100000:2.0:1767320812.765593:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6199 00000100:00100000:2.0:1767320812.765596:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985520640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320812.765605:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320812.765631:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985520640:12345-0@lo:38:kworker.0 Request processed in 33us (1035us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320812.765637:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6199 00000100:00100000:2.0:1767320812.765648:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6200 00000100:00100000:2.0:1767320812.765652:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985520896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320812.765660:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320812.765682:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985520896:12345-0@lo:38:kworker.0 Request processed in 29us (1054us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320812.765687:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6200 00000100:00080000:2.0:1767320812.765703:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320812.765914:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985520768:0@lo:400:kworker.0 00000100:00100000:2.0:1767320812.765969:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320812.766011:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320812.766017:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320812.766020:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985520256:0@lo:38:kworker.0 00000100:00080000:2.0:1767320812.766030:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320812.766034:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320812.766037:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985520384:0@lo:38:kworker.0 00000100:00080000:2.0:1767320812.766046:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320812.766049:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320812.766052:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985520640:0@lo:38:kworker.0 00000100:00080000:2.0:1767320812.766060:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320812.766063:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320812.766065:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985520896:0@lo:38:kworker.0 00000100:00100000:2.0:1767320812.766088:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985520512 00000100:00100000:2.0:1767320812.766092:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985520512 00000100:00100000:2.0:1767320812.766116:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5446 00000100:00100000:2.0:1767320812.766122:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985520512:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320812.766160:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985520512:12345-0@lo:400:kworker.0 Request processed in 38us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320812.766191:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5446 00000100:00080000:2.0:1767320812.766206:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320812.766222:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985520512:0@lo:400:kworker.0 00010000:00010000:1.0:1767320813.403514:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320814.429131:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320815.323776:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320815.323831:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985521408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320815.323924:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320815.323996:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985521408 00000100:00100000:0.0:1767320815.324009:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985521408 00000100:00080000:2.0:1767320815.324055:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320815.324092:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2742 00000100:00100000:0.0:1767320815.324105:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985521408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320815.324139:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320815.324257:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985521408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 154us (331us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320815.324265:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2742 00000100:00100000:2.0:1767320815.326006:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985521408:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320815.328737:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985521280:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320815.328848:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320815.329475:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985521280 00000100:00100000:0.0:1767320815.329491:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985521280 00000100:00100000:0.0:1767320815.329581:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2743 00000100:00100000:0.0:1767320815.329591:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985521280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320815.329626:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320815.329718:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985521280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 129us (870us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320815.329726:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2743 00000100:00080000:2.0:1767320815.329827:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320815.329925:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985521280:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320815.451878:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320816.475585:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320817.505759:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320817.840065:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320817.840105:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767320817.840135:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920638464 offset 0 length 520: 4 00000100:00100000:1.0:1767320817.840214:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920638208 00000100:00100000:1.0:1767320817.840225:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920638208 00000100:00100000:0.0:1767320817.840279:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920638336 00000100:00100000:0.0:1767320817.840285:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920638336 00000100:00100000:1.0:1767320817.840291:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6201 00000100:00100000:1.0:1767320817.840302:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920638208:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320817.840326:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320817.840346:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5447 00000100:00100000:0.0:1767320817.840357:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920638336:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320817.840388:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920638208:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 86us (332us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320817.840396:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6201 00000100:00100000:0.0:1767320817.840404:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920638336:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 48us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320817.840413:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5447 00000100:00080000:3.0:1767320817.840565:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320817.883300:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320792/real 1767320792] req@ffff8ce1b3ffc380 x1853163985515008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320817 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320817.883333:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3ffc380 x1853163985515008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320817 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320817.883351:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320817.883360:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320817.883363:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985515008:0@lo:250:kworker.0 00000100:00080000:3.0:1767320817.885232:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320817.885242:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320817.885246:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320817.885265:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6040 00000100:00080000:3.0:1767320817.885270:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320817.885277:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320817.885320:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10a80 x1853163985521536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320817.885352:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320817.885356:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320817.885358:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320817.885362:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6061 00000100:00080000:3.0:1767320817.885364:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320817.885369:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320817.885387:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12300 x1853163985521664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320817.885395:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320817.885398:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320817.885400:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320817.885403:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6061 00000100:00080000:3.0:1767320817.885405:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320817.885408:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320817.885419:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10000 x1853163985521792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320817.885465:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320817.885469:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320817.885470:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320817.885473:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6061 00000100:00080000:3.0:1767320817.885476:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320817.885479:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320817.885493:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985522048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320817.885519:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320817.885523:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320817.885524:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320817.885528:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6061 00000100:00080000:3.0:1767320817.885530:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320817.885533:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320817.885544:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11180 x1853163985522304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320817.885588:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985522560:0@lo:400:kworker.0 00000100:00100000:3.0:1767320817.885653:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320817.885696:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985521920:0@lo:400:kworker.0 00000100:00100000:3.0:1767320817.885727:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522560 00000100:00100000:0.0:1767320817.885729:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320817.885733:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522560 00000100:00100000:0.0:1767320817.885764:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985521920 00000100:00100000:0.0:1767320817.885767:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985521920 00000100:00100000:3.0:1767320817.885781:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3533 00000100:00100000:3.0:1767320817.885789:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985522560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320817.885794:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5448 00000100:00100000:0.0:1767320817.885798:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985521920:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320817.885835:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985521920:12345-0@lo:400:kworker.0 Request processed in 35us (104us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320817.885853:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5448 00000100:00100000:1.0:1767320817.886063:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985522432:0@lo:400:kworker.0 00000100:00100000:2.0:1767320817.886123:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985521536:0@lo:250:kworker.0 00000100:00100000:0.0:1767320817.889786:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985521920:0@lo:400:kworker.0 00000100:00100000:0.0:1767320817.889875:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985522176:0@lo:400:kworker.0 00000100:00100000:0.0:1767320817.889941:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320817.890007:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522176 00000100:00100000:0.0:1767320817.890014:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522176 00000100:00100000:0.0:1767320817.890066:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5449 00000100:00100000:0.0:1767320817.890074:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985522176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320817.890123:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985522176:12345-0@lo:400:kworker.0 Request processed in 49us (182us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320817.890130:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5449 00000100:00100000:0.0:1767320817.890181:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985522176:0@lo:400:kworker.0 00000100:00080000:0.0:1767320817.890209:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320817.890214:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320817.891212:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320817.891263:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522432 00000100:00100000:1.0:1767320817.891270:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522432 00000100:00100000:1.0:1767320817.891312:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3534 00000100:00100000:1.0:1767320817.891321:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985522432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320817.891365:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985522432:12345-0@lo:400:kworker.0 Request processed in 44us (158us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320817.891372:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3534 00000100:00100000:1.0:1767320817.891419:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985522432:0@lo:400:kworker.0 00000100:00080000:0.0:1767320817.891663:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000400:00000100:2.0:1767320817.895118:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985521536 offset 0 length 520: 4 00000100:00100000:2.0:1767320817.895183:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985521664:0@lo:38:kworker.0 00000100:00100000:2.0:1767320817.895225:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320817.895247:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985521792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320817.895283:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320817.895316:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985522048:0@lo:38:kworker.0 00000100:00100000:2.0:1767320817.895337:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320817.895361:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985522304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320817.895381:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320817.895432:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985521664 00000100:00100000:2.0:1767320817.895440:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985521664 00000100:00100000:2.0:1767320817.895472:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522048 00000100:00100000:2.0:1767320817.895476:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522048 00000100:00100000:2.0:1767320817.895489:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522304 00000100:00100000:2.0:1767320817.895492:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522304 00000100:00100000:2.0:1767320817.895510:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6202 00000100:00100000:2.0:1767320817.895517:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985521664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320817.895538:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320817.895579:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985521664:12345-0@lo:38:kworker.0 Request processed in 61us (353us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320817.895586:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6202 00000100:00100000:2.0:1767320817.895602:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6203 00000100:00100000:2.0:1767320817.895606:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985522048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320817.895615:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320817.895635:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985522048:12345-0@lo:38:kworker.0 Request processed in 28us (297us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320817.895640:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6203 00000100:00100000:2.0:1767320817.895652:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6204 00000100:00100000:2.0:1767320817.895655:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985522304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320817.895664:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320817.895684:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985522304:12345-0@lo:38:kworker.0 Request processed in 27us (302us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320817.895689:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6204 00000100:00080000:2.0:1767320817.895734:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320817.895739:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320817.895742:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985521664:0@lo:38:kworker.0 00000100:00080000:2.0:1767320817.895751:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320817.895754:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320817.895756:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985522048:0@lo:38:kworker.0 00000100:00080000:2.0:1767320817.895766:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320817.895769:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320817.895772:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985522304:0@lo:38:kworker.0 00000100:00100000:1.0:1767320817.896897:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985521792 00000100:00100000:1.0:1767320817.896908:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985521792 00000100:00100000:1.0:1767320817.896955:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2041 00000100:00100000:1.0:1767320817.896962:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985521792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320817.896978:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320817.897039:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985521792:12345-0@lo:38:kworker.0 Request processed in 76us (1754us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320817.897046:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2041 00000100:00080000:2.0:1767320817.897138:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320817.897143:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320817.897147:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985521792:0@lo:38:kworker.0 00000100:00100000:3.0:1767320817.899424:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985522560:12345-0@lo:400:kworker.0 Request processed in 13626us (13761us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320817.899438:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3533 00000100:00100000:2.0:1767320817.903548:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985522560:0@lo:400:kworker.0 00010000:00010000:1.0:1767320818.523407:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320819.548839:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320820.443650:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320820.443674:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985522688:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320820.443746:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320820.443844:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522688 00000100:00100000:3.0:1767320820.443853:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522688 00000100:00100000:3.0:1767320820.443924:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2744 00000100:00100000:3.0:1767320820.443934:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985522688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320820.443961:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320820.444018:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985522688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (271us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320820.444025:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2744 00000100:00100000:3.0:1767320820.444068:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985522688:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320820.444537:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320820.444717:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985522816:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320820.444758:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320820.444832:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522816 00000100:00100000:2.0:1767320820.444838:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522816 00000100:00100000:2.0:1767320820.444884:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2745 00000100:00100000:2.0:1767320820.444892:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985522816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320820.444909:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320820.444941:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985522816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (182us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320820.444948:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2745 00000100:00080000:1.0:1767320820.445084:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320820.445115:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985522816:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320820.573491:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320821.595667:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320822.619594:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320822.957617:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320822.957669:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320822.957765:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920638720 00000100:00100000:3.0:1767320822.957775:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920638720 00000100:00100000:3.0:1767320822.957860:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920638592 00000100:00100000:3.0:1767320822.957864:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920638592 00000100:00100000:3.0:1767320822.957886:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5450 00000100:00100000:3.0:1767320822.957897:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920638720:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767320822.957964:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320822.957983:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920638720:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (377us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320822.957991:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5450 00000100:00100000:3.0:1767320822.958012:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6205 00000100:00100000:3.0:1767320822.958016:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920638592:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320822.958037:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320822.958063:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920638592:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (394us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320822.958069:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6205 00000100:00080000:3.0:1767320823.004706:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320823.004717:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320823.004720:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320823.004723:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320823.004728:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320823.004735:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6066 00000100:00080000:3.0:1767320823.004740:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320823.004746:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320823.004794:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8380 x1853163985522944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320823.004824:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320823.004827:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320823.004829:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320823.004832:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6066 00000100:00080000:3.0:1767320823.004835:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320823.004838:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320823.004850:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9880 x1853163985523072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320823.004896:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320823.004901:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320823.004903:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320823.004907:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6066 00000100:00080000:3.0:1767320823.004909:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320823.004913:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320823.004982:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320823.004997:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985523200:0@lo:400:kworker.0 00000100:00100000:3.0:1767320823.005040:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9180 x1853163985523328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320823.005065:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320823.005082:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320823.005086:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320823.005089:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320823.005094:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6066 00000100:00080000:3.0:1767320823.005096:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320823.005100:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320823.005114:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdbb80 x1853163985523584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320823.005144:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985523840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320823.005186:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985523456:0@lo:400:kworker.0 00000100:00100000:3.0:1767320823.005226:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320823.005232:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320823.005260:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320823.005269:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985523712:0@lo:400:kworker.0 00000100:00100000:0.0:1767320823.005277:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523200 00000100:00100000:0.0:1767320823.005285:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523200 00000100:00100000:3.0:1767320823.005299:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320823.005328:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523840 00000100:00100000:3.0:1767320823.005466:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523840 00000100:00100000:0.0:1767320823.005487:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523456 00000100:00100000:0.0:1767320823.005493:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523456 00000100:00100000:3.0:1767320823.005504:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523712 00000100:00100000:3.0:1767320823.005510:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523712 00000100:00100000:0.0:1767320823.005527:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5451 00000100:00100000:0.0:1767320823.005537:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985523200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320823.005543:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3535 00000100:00100000:3.0:1767320823.005553:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985523840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320823.005598:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985523200:12345-0@lo:400:kworker.0 Request processed in 63us (534us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320823.005606:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5451 00000100:00100000:0.0:1767320823.005623:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5452 00000100:00100000:0.0:1767320823.005630:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985523456:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320823.005655:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985522944:0@lo:38:kworker.0 00000100:00100000:0.0:1767320823.005663:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985523456:12345-0@lo:400:kworker.0 Request processed in 33us (436us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320823.005669:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5452 00000100:00080000:0.0:1767320823.005689:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320823.005690:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320823.005694:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320823.005697:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320823.005720:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985523072:0@lo:38:kworker.0 00000100:00100000:2.0:1767320823.005744:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320823.005773:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3536 00000100:00100000:1.0:1767320823.005784:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985523712:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320823.005787:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985523328:0@lo:38:kworker.0 00000100:00100000:2.0:1767320823.005824:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320823.005831:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985523712:12345-0@lo:400:kworker.0 Request processed in 49us (531us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320823.005838:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3536 00000100:00100000:2.0:1767320823.005843:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985523584:0@lo:38:kworker.0 00000100:00100000:2.0:1767320823.005865:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320823.005891:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985522944 00000100:00100000:1.0:1767320823.005896:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985522944 00000100:00100000:2.0:1767320823.005898:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523072 00000100:00100000:2.0:1767320823.005904:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523072 00000100:00100000:1.0:1767320823.005919:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523328 00000100:00100000:1.0:1767320823.005923:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523328 00000100:00100000:1.0:1767320823.005938:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6206 00000100:00100000:1.0:1767320823.005943:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985522944:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320823.005953:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2042 00000100:00100000:2.0:1767320823.005959:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985523072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320823.005962:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767320823.005975:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320823.006005:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985522944:12345-0@lo:38:kworker.0 Request processed in 61us (315us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320823.006012:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6206 00000100:00100000:1.0:1767320823.006025:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523584 00000100:00100000:2.0:1767320823.006027:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985523072:12345-0@lo:38:kworker.0 Request processed in 65us (280us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320823.006030:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523584 00000100:00100000:2.0:1767320823.006038:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2042 00000100:00100000:1.0:1767320823.006044:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6207 00000100:00100000:1.0:1767320823.006049:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985523328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320823.006059:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320823.006066:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6208 00000100:00100000:2.0:1767320823.006072:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985523584:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320823.006084:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320823.006087:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985523328:12345-0@lo:38:kworker.0 Request processed in 34us (260us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320823.006093:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6207 00000100:00100000:2.0:1767320823.006117:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985523584:12345-0@lo:38:kworker.0 Request processed in 43us (251us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320823.006124:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6208 00000100:00080000:1.0:1767320823.011057:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320823.011071:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320823.011075:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985522944:0@lo:38:kworker.0 00000100:00080000:1.0:1767320823.011089:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320823.011092:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320823.011095:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985523072:0@lo:38:kworker.0 00000100:00080000:1.0:1767320823.011106:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320823.011109:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320823.011112:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985523328:0@lo:38:kworker.0 00000100:00080000:1.0:1767320823.011132:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320823.011135:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320823.011138:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985523584:0@lo:38:kworker.0 00000100:00100000:3.0:1767320823.012726:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985523840:12345-0@lo:400:kworker.0 Request processed in 7166us (7487us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320823.012740:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3535 00000100:00100000:0.0:1767320823.016529:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985523840:0@lo:400:kworker.0 00000100:00100000:0.0:1767320823.016557:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985523200:0@lo:400:kworker.0 00000100:00100000:1.0:1767320823.021074:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985523712:0@lo:400:kworker.0 00000100:00100000:1.0:1767320823.021095:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985523456:0@lo:400:kworker.0 00010000:00010000:0.0:1767320823.643802:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320824.667665:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320825.563733:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985523968:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320825.563808:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320825.563969:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985523968 00000100:00100000:2.0:1767320825.563977:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985523968 00000100:00100000:2.0:1767320825.564041:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2746 00000100:00100000:2.0:1767320825.564051:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985523968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320825.564081:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320825.564137:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985523968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (328us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320825.564145:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2746 00000100:00080000:0.0:1767320825.564342:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320825.564385:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985523968:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320825.567098:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320825.567136:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985524096:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320825.567237:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320825.567475:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524096 00000100:00100000:2.0:1767320825.567485:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524096 00000100:00100000:2.0:1767320825.567557:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2747 00000100:00100000:2.0:1767320825.567570:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985524096:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320825.567604:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320825.567652:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985524096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (414us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320825.567659:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2747 00000100:00080000:0.0:1767320825.568061:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320825.568102:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985524096:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320825.691418:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320826.715832:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320827.739466:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320828.075442:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320828.075519:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320828.075541:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920638976 00000100:00100000:1.0:1767320828.075552:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920638976 00000100:00100000:2.0:1767320828.075611:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920638848 00000100:00100000:2.0:1767320828.075618:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920638848 00000100:00100000:1.0:1767320828.075643:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5453 00000100:00100000:2.0:1767320828.075653:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6209 00000100:00100000:1.0:1767320828.075658:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920638976:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320828.075667:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920638848:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320828.075692:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320828.075776:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320828.075835:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920638976:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 182us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320828.075843:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5453 00000100:00100000:2.0:1767320828.081505:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920638848:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 5827us (5972us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320828.081520:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6209 00000100:00080000:1.0:1767320828.123375:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320828.123387:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320828.123390:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320828.123394:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320828.123398:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320828.123406:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320828.123410:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320828.123417:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320828.123458:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb100 x1853163985524224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320828.123487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320828.123491:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320828.123492:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320828.123496:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320828.123498:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320828.123502:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320828.123519:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af100 x1853163985524352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320828.123555:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320828.123562:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320828.123564:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320828.123567:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320828.123570:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320828.123573:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320828.123584:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985524608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320828.123602:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767320828.123612:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320828.123617:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320828.123620:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767320828.123620:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985524480:0@lo:400:kworker.0 00000100:00080000:1.0:1767320828.123629:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320828.123632:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320828.123635:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985524224:0@lo:38:kworker.0 00000100:00080000:1.0:1767320828.123638:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320828.123654:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af480 x1853163985524864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320828.123725:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985524992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320828.123809:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320828.123819:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985525120:0@lo:400:kworker.0 00000100:00100000:3.0:1767320828.124359:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320828.124435:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524480 00000100:00100000:3.0:1767320828.124444:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524480 00000100:00100000:3.0:1767320828.124499:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5454 00000100:00100000:3.0:1767320828.124507:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985524480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320828.124559:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985524480:12345-0@lo:400:kworker.0 Request processed in 52us (209us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320828.124567:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5454 00000100:00100000:3.0:1767320828.124609:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985524480:0@lo:400:kworker.0 00000100:00100000:3.0:1767320828.124656:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985524736:0@lo:400:kworker.0 00000100:00100000:2.0:1767320828.124677:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320828.124709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985524352:0@lo:38:kworker.0 00000100:00100000:2.0:1767320828.124737:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320828.124761:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985524608:0@lo:38:kworker.0 00000100:00100000:2.0:1767320828.124781:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320828.126297:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320828.127306:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985524864:0@lo:38:kworker.0 00000100:00100000:2.0:1767320828.127559:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320828.128727:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524352 00000100:00100000:1.0:1767320828.128740:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524352 00000100:00100000:1.0:1767320828.128786:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2043 00000100:00100000:1.0:1767320828.128794:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985524352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320828.128818:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320828.128874:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985524352:12345-0@lo:38:kworker.0 Request processed in 79us (4135us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320828.128881:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2043 00000100:00100000:1.0:1767320828.128909:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524224 00000100:00100000:1.0:1767320828.128913:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524224 00000100:00100000:1.0:1767320828.128932:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524608 00000100:00100000:1.0:1767320828.128935:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524608 00000100:00100000:1.0:1767320828.128947:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524864 00000100:00100000:1.0:1767320828.128950:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524864 00000100:00100000:1.0:1767320828.128965:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6210 00000100:00100000:1.0:1767320828.128969:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985524224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320828.128979:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320828.129007:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985524224:12345-0@lo:38:kworker.0 Request processed in 36us (4329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320828.129013:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6210 00000100:00100000:1.0:1767320828.129024:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6211 00000100:00100000:1.0:1767320828.129028:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985524608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320828.129046:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320828.129083:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985524608:12345-0@lo:38:kworker.0 Request processed in 53us (4300us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320828.129089:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6211 00000100:00100000:1.0:1767320828.129100:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6212 00000100:00100000:1.0:1767320828.129104:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985524864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320828.129114:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320828.129136:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985524864:12345-0@lo:38:kworker.0 Request processed in 31us (1577us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320828.129143:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6212 00000100:00080000:2.0:1767320828.130271:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320828.130283:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320828.130288:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985524864:0@lo:38:kworker.0 00000100:00080000:2.0:1767320828.130299:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320828.130303:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320828.130307:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985524224:0@lo:38:kworker.0 00000100:00080000:2.0:1767320828.130317:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320828.130320:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320828.130323:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985524352:0@lo:38:kworker.0 00000100:00080000:2.0:1767320828.130331:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320828.130335:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320828.130337:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985524608:0@lo:38:kworker.0 00000100:00100000:3.0:1767320828.130341:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524992 00000100:00100000:3.0:1767320828.130348:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524992 00000100:00100000:2.0:1767320828.130410:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320828.130456:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985524736 00000100:00100000:2.0:1767320828.130459:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985524736 00000100:00100000:3.0:1767320828.130486:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3537 00000100:00100000:3.0:1767320828.130494:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985524992:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320828.130636:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320828.130679:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985524992:12345-0@lo:400:kworker.0 Request processed in 185us (4385us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320828.130686:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3537 00000100:00100000:3.0:1767320828.130701:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985525120 00000100:00080000:0.0:1767320828.130723:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320828.130728:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320828.130787:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985525120 00000100:00100000:3.0:1767320828.130821:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3538 00000100:00100000:3.0:1767320828.130829:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985525120:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320828.130855:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320828.130871:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985525120:12345-0@lo:400:kworker.0 Request processed in 43us (235us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320828.130878:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3538 00000100:00100000:0.0:1767320828.130912:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985524992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320828.131248:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985525120:0@lo:400:kworker.0 00000100:00100000:2.0:1767320828.132427:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5455 00000100:00100000:2.0:1767320828.132446:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985524736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320828.132520:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985524736:12345-0@lo:400:kworker.0 Request processed in 83us (2109us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320828.132528:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5455 00000100:00100000:1.0:1767320828.135585:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985524736:0@lo:400:kworker.0 00010000:00010000:2.0:1767320828.763549:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320829.788130:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320830.683578:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320830.683603:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985525248:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320830.683671:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320830.684292:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985525248 00000100:00100000:2.0:1767320830.684303:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985525248 00000100:00100000:2.0:1767320830.684383:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2748 00000100:00100000:2.0:1767320830.684397:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985525248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320830.684439:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320830.684519:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985525248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 126us (847us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320830.684527:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2748 00000100:00100000:3.0:1767320830.684718:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985525248:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320830.684943:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320830.686699:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320830.687624:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985525376:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320830.687700:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320830.687923:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985525376 00000100:00100000:3.0:1767320830.687932:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985525376 00000100:00100000:3.0:1767320830.688057:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2749 00000100:00100000:3.0:1767320830.688066:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985525376:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320830.688097:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320830.688190:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985525376:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (461us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320830.688198:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2749 00000100:00100000:0.0:1767320830.688707:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985525376:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320830.688957:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320830.813375:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320831.838554:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320832.861230:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320833.193647:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320833.194065:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920639232 00000100:00100000:2.0:1767320833.194075:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920639232 00000100:00100000:2.0:1767320833.194202:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5456 00000100:00100000:2.0:1767320833.194216:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920639232:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320833.194304:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920639232:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (666us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320833.194312:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5456 00000100:00080000:0.0:1767320833.194503:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320833.194862:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320833.195090:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920639104 00000100:00100000:3.0:1767320833.195098:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920639104 00000100:00100000:3.0:1767320833.195225:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6213 00000100:00100000:3.0:1767320833.195233:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920639104:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320833.195259:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320833.195313:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920639104:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 79us (451us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320833.195320:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6213 00000100:00080000:0.0:1767320833.243553:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320833.243564:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320833.243568:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320833.243572:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320833.243576:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320833.243584:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6076 00000100:00080000:0.0:1767320833.243589:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320833.243595:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320833.243634:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11180 x1853163985525504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320833.243689:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985525504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320833.243752:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320833.244049:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985525504 00000100:00100000:3.0:1767320833.244057:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985525504 00000100:00100000:3.0:1767320833.244108:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6214 00000100:00100000:3.0:1767320833.244115:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985525504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320833.244134:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320833.244244:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320833.244248:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320833.244250:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320833.244253:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6076 00000100:00080000:0.0:1767320833.244256:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320833.244260:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320833.244289:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11c00 x1853163985525632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320833.249599:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985525504:12345-0@lo:38:kworker.0 Request processed in 5471us (5834us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320833.249613:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6214 00000100:00080000:2.0:1767320833.249792:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320833.249799:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320833.249804:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985525504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320833.249846:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985525632:0@lo:38:kworker.0 00000100:00100000:2.0:1767320833.250043:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320833.250355:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320833.250359:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320833.250362:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320833.250367:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6076 00000100:00080000:0.0:1767320833.250371:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320833.250377:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320833.250396:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13100 x1853163985525888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320833.250422:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320833.250425:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320833.250427:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320833.250430:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6076 00000100:00080000:0.0:1767320833.250432:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320833.250435:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320833.250445:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11880 x1853163985526144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320833.250484:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985526400:0@lo:400:kworker.0 00000100:00100000:0.0:1767320833.250519:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320833.250544:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320833.250550:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985525760:0@lo:400:kworker.0 00000100:00100000:0.0:1767320833.250576:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320833.250598:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985526016:0@lo:400:kworker.0 00000100:00100000:0.0:1767320833.250618:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320833.250636:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320833.250641:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985526272:0@lo:400:kworker.0 00000100:00100000:0.0:1767320833.250661:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320833.250686:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526400 00000100:00100000:0.0:1767320833.250692:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526400 00000100:00100000:0.0:1767320833.250739:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526272 00000100:00100000:0.0:1767320833.250742:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526272 00000100:00100000:0.0:1767320833.250765:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3539 00000100:00100000:0.0:1767320833.250772:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985526400:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320833.250827:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985526400:12345-0@lo:400:kworker.0 Request processed in 55us (307us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320833.250835:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3539 00000100:00100000:0.0:1767320833.250849:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3540 00000100:00100000:0.0:1767320833.250856:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985526272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320833.251054:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985525888:0@lo:38:kworker.0 00000100:00100000:2.0:1767320833.251087:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320833.251107:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985526144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320833.251127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320833.251189:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985526272:12345-0@lo:400:kworker.0 Request processed in 308us (501us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320833.251197:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3540 00000100:00080000:2.0:1767320833.251204:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320833.251209:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320833.251242:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985526272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320833.251243:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985525760 00000100:00100000:2.0:1767320833.251249:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985525760 00000100:00100000:0.0:1767320833.251273:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985526400:0@lo:400:kworker.0 00000100:00100000:2.0:1767320833.251290:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526016 00000100:00100000:2.0:1767320833.251295:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526016 00000100:00100000:0.0:1767320833.251306:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985525888 00000100:00100000:0.0:1767320833.251311:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985525888 00000100:00100000:2.0:1767320833.251320:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526144 00000100:00100000:2.0:1767320833.251324:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526144 00000100:00100000:0.0:1767320833.251332:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5457 00000100:00100000:0.0:1767320833.251341:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985525760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320833.251347:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5458 00000100:00100000:2.0:1767320833.251357:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985526016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320833.251389:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985525760:12345-0@lo:400:kworker.0 Request processed in 50us (811us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320833.251396:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5457 00000100:00100000:2.0:1767320833.251414:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985526016:12345-0@lo:400:kworker.0 Request processed in 58us (793us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320833.251418:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6215 00000100:00100000:0.0:1767320833.251423:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985525888:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320833.251439:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5458 00010000:00020000:0.0:1767320833.251441:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320833.251459:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6216 00000100:00100000:2.0:1767320833.251465:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985526144:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320833.251478:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985525888:12345-0@lo:38:kworker.0 Request processed in 52us (389us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767320833.251479:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320833.251489:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6215 00000100:00100000:0.0:1767320833.251517:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985526016:0@lo:400:kworker.0 00000100:00100000:2.0:1767320833.251520:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985526144:12345-0@lo:38:kworker.0 Request processed in 51us (389us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320833.251529:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6216 00000100:00100000:0.0:1767320833.251541:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985525760:0@lo:400:kworker.0 00000100:00080000:2.0:1767320833.251555:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320833.251566:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320833.251571:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320833.251574:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985525888:0@lo:38:kworker.0 00000100:00080000:0.0:1767320833.251586:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320833.251589:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320833.251593:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985526144:0@lo:38:kworker.0 00000100:00100000:1.0:1767320833.252657:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985525632 00000100:00100000:1.0:1767320833.252668:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985525632 00000100:00100000:1.0:1767320833.252710:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2044 00000100:00100000:1.0:1767320833.252716:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985525632:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320833.252733:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320833.252914:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985525632:12345-0@lo:38:kworker.0 Request processed in 196us (2870us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320833.252921:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2044 00000100:00080000:0.0:1767320833.256656:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320833.256669:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320833.256674:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985525632:0@lo:38:kworker.0 00010000:00010000:2.0:1767320833.883393:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320834.907610:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320835.803746:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985526528:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320835.803815:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320835.803960:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526528 00000100:00100000:2.0:1767320835.803969:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526528 00000100:00100000:2.0:1767320835.804076:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2750 00000100:00100000:2.0:1767320835.804086:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985526528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320835.804117:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320835.804161:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985526528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (345us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320835.804200:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2750 00000100:00100000:2.0:1767320835.804269:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985526528:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320835.804429:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320835.804785:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985526656:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320835.804821:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320835.804893:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526656 00000100:00100000:1.0:1767320835.804898:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526656 00000100:00100000:1.0:1767320835.804967:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2751 00000100:00100000:1.0:1767320835.804975:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985526656:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320835.804996:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320835.805030:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985526656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (208us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320835.805035:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2751 00000100:00100000:1.0:1767320835.805066:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985526656:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320835.805476:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767320835.932259:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320836.958728:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320837.980106:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320838.318025:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320838.318069:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320838.318142:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920639488 00000100:00100000:3.0:1767320838.318152:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920639488 00000100:00100000:3.0:1767320838.318307:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920639360 00000100:00100000:3.0:1767320838.318311:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920639360 00000100:00080000:2.0:1767320838.318412:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320838.318603:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5459 00000100:00100000:1.0:1767320838.318616:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920639488:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320838.318694:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920639488:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (678us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320838.318702:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5459 00000100:00100000:1.0:1767320838.318723:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6217 00000100:00100000:1.0:1767320838.318727:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920639360:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320838.318746:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320838.318780:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920639360:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (711us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320838.318787:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6217 00000100:00080000:2.0:1767320838.363416:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320838.363426:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320838.363430:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320838.363435:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320838.363439:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320838.363448:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6081 00000100:00080000:2.0:1767320838.363454:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320838.363462:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320838.363507:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1f80 x1853163985526784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320838.363532:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320838.363536:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320838.363538:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320838.363541:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6081 00000100:00080000:2.0:1767320838.363543:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320838.363546:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320838.363559:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1180 x1853163985526912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320838.363603:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320838.363607:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320838.363609:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320838.363622:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6081 00000100:00080000:2.0:1767320838.363625:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320838.363628:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320838.363641:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985527168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320838.363669:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320838.363673:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320838.363674:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320838.363678:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6081 00000100:00080000:2.0:1767320838.363680:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320838.363683:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320838.363695:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1500 x1853163985527424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320838.363747:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985527680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320838.363809:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320838.363867:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320838.363873:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985527040:0@lo:400:kworker.0 00000100:00100000:2.0:1767320838.363906:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320838.363940:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985527552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320838.363962:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320838.363976:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320838.363981:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985527296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320838.363999:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320838.364041:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527040 00000100:00100000:2.0:1767320838.364049:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527040 00000100:00100000:2.0:1767320838.364098:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527296 00000100:00100000:2.0:1767320838.364101:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527296 00000100:00100000:2.0:1767320838.364122:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5460 00000100:00100000:2.0:1767320838.364130:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985527040:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320838.364209:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985527040:12345-0@lo:400:kworker.0 Request processed in 79us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320838.364217:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5460 00000100:00100000:2.0:1767320838.364233:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5461 00000100:00100000:2.0:1767320838.364239:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985527296:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320838.364267:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985527296:12345-0@lo:400:kworker.0 Request processed in 28us (266us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320838.364272:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5461 00000100:00100000:2.0:1767320838.364321:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985527040:0@lo:400:kworker.0 00000100:00080000:2.0:1767320838.364341:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320838.364355:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985527296:0@lo:400:kworker.0 00000100:00100000:3.0:1767320838.364474:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527680 00000100:00100000:3.0:1767320838.364479:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527680 00000100:00100000:3.0:1767320838.364525:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527552 00000100:00100000:3.0:1767320838.364529:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527552 00000100:00100000:3.0:1767320838.364550:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3541 00000100:00100000:3.0:1767320838.364557:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985527680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320838.364596:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985527680:12345-0@lo:400:kworker.0 Request processed in 40us (786us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320838.364603:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3541 00000100:00100000:3.0:1767320838.364615:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3542 00000100:00100000:3.0:1767320838.364621:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985527552:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320838.364649:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985527552:12345-0@lo:400:kworker.0 Request processed in 28us (686us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320838.364655:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3542 00000100:00100000:3.0:1767320838.364693:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985527680:0@lo:400:kworker.0 00000100:00100000:3.0:1767320838.364713:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985527552:0@lo:400:kworker.0 00000100:00100000:1.0:1767320838.364819:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985526784:0@lo:38:kworker.0 00000100:00100000:1.0:1767320838.364863:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.364895:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526784 00000100:00100000:0.0:1767320838.364901:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526784 00000100:00100000:1.0:1767320838.364906:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985526912:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.364927:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6218 00000100:00100000:0.0:1767320838.364933:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985526784:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320838.364937:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767320838.364955:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320838.364967:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985527168:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.365001:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985526784:12345-0@lo:38:kworker.0 Request processed in 65us (136us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320838.365001:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.365010:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6218 00000100:00100000:0.0:1767320838.365025:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527168 00000100:00100000:1.0:1767320838.365025:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985527424:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.365031:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527168 00000100:00100000:1.0:1767320838.365054:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.365058:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6219 00000100:00100000:0.0:1767320838.365064:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985527168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320838.365074:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320838.365086:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320838.365093:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320838.365097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985526784:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.365106:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985527168:12345-0@lo:38:kworker.0 Request processed in 39us (105us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320838.365108:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320838.365115:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320838.365119:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6219 00000100:00100000:1.0:1767320838.365121:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985527168:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.365133:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527424 00000100:00100000:0.0:1767320838.365138:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527424 00000100:00100000:1.0:1767320838.365152:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985526912 00000100:00100000:1.0:1767320838.365157:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985526912 00000100:00100000:0.0:1767320838.365160:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6220 00000100:00100000:0.0:1767320838.365183:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985527424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320838.365192:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320838.365220:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985527424:12345-0@lo:38:kworker.0 Request processed in 36us (167us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320838.365225:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6220 00000100:00080000:0.0:1767320838.365445:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320838.365449:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320838.365451:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985527424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320838.365998:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2045 00000100:00100000:1.0:1767320838.366006:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985526912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320838.366022:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320838.366067:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985526912:12345-0@lo:38:kworker.0 Request processed in 60us (1131us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320838.366075:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2045 00000100:00080000:0.0:1767320838.366093:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320838.366100:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320838.366103:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985526912:0@lo:38:kworker.0 00000100:00080000:2.0:1767320838.366151:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320838.366157:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320839.003409:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320840.028736:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320840.923349:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320840.923371:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985527936:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320840.923442:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320840.923555:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527936 00000100:00100000:2.0:1767320840.923563:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527936 00000100:00100000:2.0:1767320840.923632:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2752 00000100:00100000:2.0:1767320840.923641:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985527936:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320840.923669:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320840.923709:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985527936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (267us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320840.923716:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2752 00000100:00080000:2.0:1767320840.923735:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320840.923778:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320840.923791:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985527808:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320840.923825:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320840.923897:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985527808 00000100:00100000:1.0:1767320840.923902:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985527808 00000100:00100000:1.0:1767320840.923933:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2753 00000100:00100000:1.0:1767320840.923941:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985527808:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320840.923961:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320840.923981:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985527936:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320840.924021:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985527808:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (195us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320840.924028:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2753 00000100:00100000:1.0:1767320840.924061:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985527808:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320840.924132:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320841.051412:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320842.075474:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767320842.416217:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920639616 offset 0 length 520: 4 00000100:00100000:0.0:1767320842.416267:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320842.416360:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920639744 00000100:00100000:3.0:1767320842.416371:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920639744 00000100:00100000:3.0:1767320842.416425:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6221 00000100:00100000:3.0:1767320842.416434:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920639744:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320842.416457:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320842.416529:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920639744:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (261us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320842.416537:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6221 00000100:00100000:2.0:1767320842.416858:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320842.416916:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920639872 00000100:00100000:3.0:1767320842.416921:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920639872 00000100:00100000:3.0:1767320842.417008:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5462 00000100:00100000:3.0:1767320842.417016:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920639872:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320842.417081:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920639872:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 65us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320842.417088:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5462 00000100:00080000:2.0:1767320842.426823:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320842.459263:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320817/real 1767320817] req@ffff8ce184a10a80 x1853163985521536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320842 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320842.459295:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce184a10a80 x1853163985521536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320842 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320842.459314:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320842.459335:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320842.459339:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985521536:0@lo:250:kworker.0 00000100:00080000:0.0:1767320842.459483:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320842.459488:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320842.459491:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320842.459496:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6066 00000100:00080000:0.0:1767320842.459500:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320842.460129:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320842.460191:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74a80 x1853163985528064/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320842.460212:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320842.460215:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320842.460217:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320842.460222:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6086 00000100:00080000:0.0:1767320842.460224:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320842.460227:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320842.460241:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75180 x1853163985528192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320842.460249:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320842.460251:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320842.460254:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320842.460256:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6086 00000100:00080000:0.0:1767320842.460259:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320842.460261:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320842.460273:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10a80 x1853163985528320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320842.460306:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320842.460310:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320842.460311:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320842.460315:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6086 00000100:00080000:0.0:1767320842.460317:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320842.460320:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320842.460331:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11880 x1853163985528576/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320842.460354:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985528448:0@lo:400:kworker.0 00000100:00080000:0.0:1767320842.460357:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320842.460363:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320842.460365:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320842.460368:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6086 00000100:00080000:0.0:1767320842.460370:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320842.460373:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320842.460384:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12300 x1853163985528832/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320842.460438:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320842.460506:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320842.460514:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985529088:0@lo:400:kworker.0 00000100:00100000:3.0:1767320842.460542:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320842.460576:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529088 00000100:00100000:3.0:1767320842.460584:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529088 00000100:00100000:3.0:1767320842.460639:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3543 00000100:00100000:3.0:1767320842.460650:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985529088:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320842.460701:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985529088:12345-0@lo:400:kworker.0 Request processed in 54us (157us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320842.460709:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3543 00000100:00100000:3.0:1767320842.460735:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985528448 00000100:00100000:3.0:1767320842.460739:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985528448 00000100:00100000:3.0:1767320842.460780:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5463 00000100:00100000:3.0:1767320842.460788:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985528448:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320842.460809:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320842.460815:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320842.460836:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985528448:12345-0@lo:400:kworker.0 Request processed in 49us (399us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320842.460839:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985528064:0@lo:250:kworker.0 00000100:00100000:3.0:1767320842.460849:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5463 00000400:00000100:2.0:1767320842.460870:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985528064 offset 0 length 520: 4 00000100:00100000:3.0:1767320842.460892:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985529088:0@lo:400:kworker.0 00000100:00100000:2.0:1767320842.460894:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985528192:0@lo:38:kworker.0 00000100:00100000:3.0:1767320842.460911:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985528448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320842.460926:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320842.461056:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985528960:0@lo:400:kworker.0 00000100:00100000:0.0:1767320842.461085:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320842.461106:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320842.461112:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985528704:0@lo:400:kworker.0 00000100:00100000:0.0:1767320842.461131:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320842.461267:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985528320:0@lo:38:kworker.0 00000100:00100000:2.0:1767320842.461295:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320842.461313:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985528576:0@lo:38:kworker.0 00000100:00100000:2.0:1767320842.461341:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985528320 00000100:00100000:2.0:1767320842.461345:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985528320 00000100:00100000:2.0:1767320842.461369:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2046 00000100:00100000:2.0:1767320842.461375:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985528320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320842.461393:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320842.461499:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985528320:12345-0@lo:38:kworker.0 Request processed in 122us (202us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320842.461518:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2046 00000100:00100000:2.0:1767320842.461542:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320842.465570:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985528192 00000100:00100000:3.0:1767320842.465585:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985528192 00000100:00100000:3.0:1767320842.465641:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985528704 00000100:00100000:3.0:1767320842.465645:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985528704 00000100:00100000:3.0:1767320842.465699:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5464 00000100:00100000:3.0:1767320842.465708:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985528704:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320842.466558:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985528704:12345-0@lo:400:kworker.0 Request processed in 846us (5419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320842.466568:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5464 00000100:00100000:3.0:1767320842.466598:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6222 00000100:00100000:3.0:1767320842.466603:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985528192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320842.466627:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320842.466661:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985528192:12345-0@lo:38:kworker.0 Request processed in 56us (5736us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320842.466668:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6222 00000100:00100000:3.0:1767320842.466879:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985528960 00000100:00100000:3.0:1767320842.466887:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985528960 00000100:00100000:3.0:1767320842.466925:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3544 00000100:00100000:3.0:1767320842.466932:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985528960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320842.466967:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985528960:12345-0@lo:400:kworker.0 Request processed in 36us (5882us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320842.466973:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3544 00000100:00100000:3.0:1767320842.467021:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985528704:0@lo:400:kworker.0 00000100:00100000:3.0:1767320842.467036:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985528960:0@lo:400:kworker.0 00000100:00080000:3.0:1767320842.467080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320842.467086:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320842.472429:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985528832:0@lo:38:kworker.0 00000100:00100000:2.0:1767320842.472499:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320842.482067:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320842.482081:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320842.482087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985528192:0@lo:38:kworker.0 00000100:00080000:2.0:1767320842.482100:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320842.482103:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320842.482107:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985528320:0@lo:38:kworker.0 00000100:00100000:0.0:1767320842.482408:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985528576 00000100:00100000:0.0:1767320842.482416:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985528576 00000100:00100000:0.0:1767320842.482461:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985528832 00000100:00100000:0.0:1767320842.482464:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985528832 00000100:00100000:0.0:1767320842.482484:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6223 00000100:00100000:0.0:1767320842.482491:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985528576:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320842.482574:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6224 00000100:00100000:2.0:1767320842.482581:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985528832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320842.482643:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320842.482716:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985528576:12345-0@lo:38:kworker.0 Request processed in 224us (21172us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320842.482724:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6223 00000100:00080000:3.0:1767320842.482756:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320842.482763:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320842.482767:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985528576:0@lo:38:kworker.0 00010000:00020000:2.0:1767320842.485157:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320842.485263:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985528832:12345-0@lo:38:kworker.0 Request processed in 2679us (12762us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320842.485275:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6224 00000100:00080000:3.0:1767320842.485340:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320842.485347:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320842.485352:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985528832:0@lo:38:kworker.0 00010000:00010000:1.0:1767320843.101200:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320844.124637:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320845.147557:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320846.043380:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320846.043405:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985529216:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320846.043485:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320846.043693:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529216 00000100:00100000:1.0:1767320846.043703:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529216 00000100:00100000:1.0:1767320846.043788:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2754 00000100:00100000:1.0:1767320846.043798:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985529216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320846.043829:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320846.043883:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985529216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (397us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320846.043891:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2754 00000100:00100000:1.0:1767320846.043946:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985529216:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320846.044662:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320846.044676:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985529344:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320846.044718:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320846.045012:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529344 00000100:00100000:2.0:1767320846.045159:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529344 00000100:00100000:2.0:1767320846.045239:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2755 00000100:00100000:2.0:1767320846.045248:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985529344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320846.045272:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320846.045316:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985529344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 69us (597us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320846.045323:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2755 00000100:00100000:0.0:1767320846.045430:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985529344:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320846.045692:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320846.045698:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320846.171922:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320847.195635:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320847.533429:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320847.533543:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320847.533699:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640128 00000100:00100000:3.0:1767320847.533718:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640128 00000100:00100000:3.0:1767320847.533864:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640000 00000100:00100000:3.0:1767320847.533868:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640000 00000100:00100000:3.0:1767320847.533902:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5465 00000100:00100000:3.0:1767320847.533914:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920640128:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320847.533996:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920640128:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (575us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320847.534004:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5465 00000100:00100000:3.0:1767320847.534037:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6225 00000100:00100000:3.0:1767320847.534042:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920640000:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320847.534062:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320847.534089:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920640000:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (547us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320847.534099:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6225 00000100:00080000:2.0:1767320847.542881:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320847.579375:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320847.579385:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320847.579388:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320847.579392:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320847.579396:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320847.579405:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6090 00000100:00080000:0.0:1767320847.579410:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320847.579417:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320847.579460:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11880 x1853163985529472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320847.579489:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320847.579493:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320847.579495:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320847.579498:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6090 00000100:00080000:0.0:1767320847.579500:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320847.579503:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320847.579514:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10a80 x1853163985529600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320847.579566:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320847.579570:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320847.579572:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320847.579575:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6090 00000100:00080000:0.0:1767320847.579578:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320847.579581:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320847.579592:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13100 x1853163985529856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320847.579625:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320847.579629:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320847.579631:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320847.579637:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6090 00000100:00100000:3.0:1767320847.579639:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985529472:0@lo:38:kworker.0 00000100:00080000:0.0:1767320847.579640:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320847.579645:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320847.579658:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61f80 x1853163985530112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320847.579700:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985529984:0@lo:400:kworker.0 00000100:00100000:3.0:1767320847.579722:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320847.579766:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320847.579792:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985529600:0@lo:38:kworker.0 00000100:00100000:0.0:1767320847.579797:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320847.579822:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320847.579852:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985529856:0@lo:38:kworker.0 00000100:00100000:2.0:1767320847.579867:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985530368:0@lo:400:kworker.0 00000100:00100000:3.0:1767320847.579874:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320847.579898:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320847.579914:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985530112:0@lo:38:kworker.0 00000100:00100000:2.0:1767320847.579921:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320847.579928:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985529728:0@lo:400:kworker.0 00000100:00100000:2.0:1767320847.579951:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320847.579961:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985530240:0@lo:400:kworker.0 00000100:00100000:3.0:1767320847.579970:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529472 00000100:00100000:3.0:1767320847.579978:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529472 00000100:00100000:2.0:1767320847.579983:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529600 00000100:00100000:2.0:1767320847.579988:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529600 00000100:00100000:0.0:1767320847.579997:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320847.580017:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2047 00000100:00100000:2.0:1767320847.580022:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985529600:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320847.580039:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320847.580150:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529984 00000100:00100000:1.0:1767320847.580153:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529984 00000100:00100000:1.0:1767320847.580197:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529856 00000100:00100000:1.0:1767320847.580199:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529856 00000100:00100000:1.0:1767320847.580207:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985529728 00000100:00100000:1.0:1767320847.580209:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985529728 00000100:00100000:1.0:1767320847.580219:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5466 00000100:00100000:1.0:1767320847.580225:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985529984:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320847.580251:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5467 00000100:00100000:3.0:1767320847.580262:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985529728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320847.580332:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985529600:12345-0@lo:38:kworker.0 Request processed in 308us (509us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320847.580338:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2047 00000100:00100000:2.0:1767320847.580357:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985530368 00000100:00100000:2.0:1767320847.580360:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985530368 00000100:00100000:2.0:1767320847.580377:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985530240 00000100:00100000:2.0:1767320847.580378:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985530240 00000100:00100000:2.0:1767320847.580389:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3545 00000100:00100000:2.0:1767320847.580392:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985530368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320847.580423:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985530368:12345-0@lo:400:kworker.0 Request processed in 29us (524us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320847.580429:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3545 00000100:00100000:2.0:1767320847.580440:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3546 00000100:00100000:2.0:1767320847.580445:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985530240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320847.580486:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985530240:12345-0@lo:400:kworker.0 Request processed in 41us (490us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320847.580493:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3546 00000100:00100000:2.0:1767320847.580513:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6226 00000100:00100000:2.0:1767320847.580517:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985529856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320847.580525:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320847.580537:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985530240:0@lo:400:kworker.0 00000100:00100000:2.0:1767320847.580561:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985529856:12345-0@lo:38:kworker.0 Request processed in 41us (686us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320847.580567:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6226 00000100:00100000:0.0:1767320847.580581:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6227 00000100:00100000:0.0:1767320847.580594:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985529472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320847.580606:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320847.580616:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985530368:0@lo:400:kworker.0 00000100:00080000:2.0:1767320847.580630:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320847.580641:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985529472:12345-0@lo:38:kworker.0 Request processed in 47us (919us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320847.580649:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6227 00000100:00100000:2.0:1767320847.580654:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320847.580693:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985530112 00000100:00100000:0.0:1767320847.580698:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985530112 00000100:00080000:2.0:1767320847.580702:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320847.580717:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320847.580721:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985529472:0@lo:38:kworker.0 00000100:00080000:2.0:1767320847.580732:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320847.580735:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320847.580738:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985529600:0@lo:38:kworker.0 00000100:00080000:2.0:1767320847.580746:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320847.580749:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320847.580751:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985529856:0@lo:38:kworker.0 00000100:00100000:2.0:1767320847.580780:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6228 00000100:00100000:2.0:1767320847.580783:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985530112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320847.580791:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320847.580812:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985530112:12345-0@lo:38:kworker.0 Request processed in 28us (160us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320847.580816:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6228 00000100:00100000:1.0:1767320847.581145:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985529984:12345-0@lo:400:kworker.0 Request processed in 920us (1380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320847.581150:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5466 00000100:00100000:0.0:1767320847.582273:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985529984:0@lo:400:kworker.0 00000100:00080000:0.0:1767320847.582318:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320847.582323:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320847.582327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985530112:0@lo:38:kworker.0 00000100:00100000:3.0:1767320847.584223:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985529728:12345-0@lo:400:kworker.0 Request processed in 3955us (4261us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320847.584238:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5467 00000100:00100000:0.0:1767320847.584845:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985529728:0@lo:400:kworker.0 00010000:00010000:0.0:1767320848.220419:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320849.244274:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320850.267403:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320851.163481:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985530496:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320851.163541:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320851.163614:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985530624:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320851.164399:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320851.170311:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985530624 00000100:00100000:1.0:1767320851.170320:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985530496 00000100:00100000:0.0:1767320851.170327:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985530624 00000100:00100000:1.0:1767320851.170327:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985530496 00000100:00100000:0.0:1767320851.170462:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2756 00000100:00100000:0.0:1767320851.170473:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985530624:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320851.170503:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320851.170593:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985530624:12345-0@lo:13:osp-pre-1-1.0 Request processed in 122us (6196us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320851.170600:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2756 00000100:00100000:3.0:1767320851.170804:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985530624:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320851.173534:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320851.173554:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320851.173567:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2757 00000100:00100000:1.0:1767320851.173579:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985530496:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320851.173607:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320851.173703:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985530496:12345-0@lo:13:osp-pre-0-1.0 Request processed in 128us (10160us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320851.173711:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2757 00000100:00100000:0.0:1767320851.173869:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985530496:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320851.295836:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320852.320814:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320852.651128:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320852.651189:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320852.651311:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640384 00000100:00100000:2.0:1767320852.651321:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640384 00000100:00100000:2.0:1767320852.651485:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640256 00000100:00100000:2.0:1767320852.651489:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640256 00000100:00100000:2.0:1767320852.651524:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5468 00000100:00100000:2.0:1767320852.651536:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920640384:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320852.651602:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920640384:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (483us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320852.651609:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5468 00000100:00100000:2.0:1767320852.651637:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6229 00000100:00100000:2.0:1767320852.651641:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920640256:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320852.651660:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320852.651685:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920640256:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (495us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320852.651692:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6229 00000100:00080000:0.0:1767320852.651936:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320852.699665:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320852.699676:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320852.699680:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320852.699684:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320852.699688:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320852.699701:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6095 00000100:00080000:0.0:1767320852.699706:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320852.699713:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320852.699750:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63b80 x1853163985530752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320852.699775:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320852.699778:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320852.699780:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320852.699783:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6095 00000100:00080000:0.0:1767320852.699786:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320852.699789:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320852.699802:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60a80 x1853163985530880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320852.699847:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767320852.699850:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985530752:0@lo:38:kworker.0 00000100:00080000:0.0:1767320852.699851:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320852.699853:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320852.699856:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6095 00000100:00080000:0.0:1767320852.699858:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320852.699861:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320852.699871:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62300 x1853163985531136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320852.699894:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320852.699898:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320852.699899:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320852.699902:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6095 00000100:00080000:0.0:1767320852.699903:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320852.699906:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320852.699912:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320852.699914:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63100 x1853163985531392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320852.699957:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985531520:0@lo:400:kworker.0 00000100:00100000:2.0:1767320852.699975:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985530880:0@lo:38:kworker.0 00000100:00100000:0.0:1767320852.699987:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320852.700008:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320852.700008:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320852.700013:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985531264:0@lo:400:kworker.0 00000100:00100000:2.0:1767320852.700032:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985531136:0@lo:38:kworker.0 00000100:00100000:0.0:1767320852.700033:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320852.700050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320852.700063:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985531392:0@lo:38:kworker.0 00000100:00100000:0.0:1767320852.700070:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531520 00000100:00100000:0.0:1767320852.700077:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531520 00000100:00100000:2.0:1767320852.700080:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320852.700108:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985530880 00000100:00100000:2.0:1767320852.700112:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985530880 00000100:00100000:0.0:1767320852.700121:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3547 00000100:00100000:0.0:1767320852.700129:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985531520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320852.700129:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2048 00000100:00100000:2.0:1767320852.700133:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985530880:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320852.700182:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320852.700191:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985531520:12345-0@lo:400:kworker.0 Request processed in 62us (202us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320852.700198:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3547 00000100:00100000:2.0:1767320852.700212:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985530880:12345-0@lo:38:kworker.0 Request processed in 77us (203us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320852.700218:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2048 00000100:00100000:0.0:1767320852.700239:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985531520:0@lo:400:kworker.0 00000100:00080000:2.0:1767320852.700253:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320852.700258:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320852.700261:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985530880:0@lo:38:kworker.0 00000100:00100000:0.0:1767320852.700270:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985530752 00000100:00100000:0.0:1767320852.700274:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985530752 00000100:00100000:0.0:1767320852.700296:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531264 00000100:00100000:0.0:1767320852.700299:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531264 00000100:00100000:2.0:1767320852.700304:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531136 00000100:00100000:2.0:1767320852.700307:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531136 00000100:00100000:0.0:1767320852.700320:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531392 00000100:00100000:0.0:1767320852.700323:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531392 00000100:00100000:2.0:1767320852.700323:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5469 00000100:00100000:2.0:1767320852.700330:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985531264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320852.700344:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6230 00000100:00100000:0.0:1767320852.700349:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985530752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320852.700359:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320852.700362:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985531264:12345-0@lo:400:kworker.0 Request processed in 35us (329us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320852.700369:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5469 00000100:00100000:2.0:1767320852.700382:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6231 00000100:00100000:2.0:1767320852.700386:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985531136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320852.700395:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320852.700465:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985531008:0@lo:400:kworker.0 00000100:00100000:1.0:1767320852.700491:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320852.700507:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320852.700512:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985531648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320852.700530:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320852.700551:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531648 00000100:00100000:1.0:1767320852.700555:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531648 00000100:00100000:1.0:1767320852.700578:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3548 00000100:00100000:1.0:1767320852.700584:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985531648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320852.700634:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985531648:12345-0@lo:400:kworker.0 Request processed in 51us (103us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320852.700641:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3548 00000100:00100000:3.0:1767320852.700655:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531008 00000100:00100000:1.0:1767320852.700657:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6232 00000100:00100000:3.0:1767320852.700659:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531008 00000100:00100000:1.0:1767320852.700661:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985531392:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320852.700671:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320852.700678:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5470 00000100:00100000:3.0:1767320852.700684:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985531008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320852.700721:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985530752:12345-0@lo:38:kworker.0 Request processed in 370us (807us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320852.700727:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6230 00000100:00100000:0.0:1767320852.700756:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985531264:0@lo:400:kworker.0 00000100:00080000:0.0:1767320852.700771:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320852.700804:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985531648:0@lo:400:kworker.0 00000100:00100000:3.0:1767320852.700950:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985531008:12345-0@lo:400:kworker.0 Request processed in 265us (456us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320852.700956:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5470 00000100:00100000:3.0:1767320852.700983:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985531008:0@lo:400:kworker.0 00000100:00100000:2.0:1767320852.701110:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985531136:12345-0@lo:38:kworker.0 Request processed in 721us (1058us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320852.701116:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6231 00000100:00100000:1.0:1767320852.701749:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985531392:12345-0@lo:38:kworker.0 Request processed in 1083us (1664us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320852.701820:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6232 00000100:00080000:0.0:1767320852.702541:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320852.702548:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320852.702551:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985531136:0@lo:38:kworker.0 00000100:00080000:0.0:1767320852.702562:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320852.702566:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320852.702569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985531392:0@lo:38:kworker.0 00000100:00080000:0.0:1767320852.702577:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320852.702580:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320852.702582:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985530752:0@lo:38:kworker.0 00010000:00010000:2.0:1767320853.339422:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320854.363394:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320855.387743:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320856.283870:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985531904:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320856.283886:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320856.283897:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985531776:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320856.283947:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320856.283996:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320856.284078:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531904 00000100:00100000:3.0:1767320856.284086:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531904 00000100:00100000:3.0:1767320856.284150:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985531776 00000100:00100000:3.0:1767320856.284153:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985531776 00000100:00100000:3.0:1767320856.284203:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2758 00000100:00100000:3.0:1767320856.284212:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985531904:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320856.284243:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320856.284285:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985531904:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (339us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320856.284292:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2758 00000100:00100000:3.0:1767320856.284308:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2759 00000100:00100000:3.0:1767320856.284314:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb2450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985531776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320856.284327:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320856.284354:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb2450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985531776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 41us (358us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320856.284371:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2759 00000100:00100000:2.0:1767320856.284486:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985531776:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320856.284971:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985531904:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320856.284990:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320856.284995:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320856.411486:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320857.435442:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320857.769815:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320857.770970:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320857.771304:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640640 00000100:00100000:2.0:1767320857.771315:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640640 00000100:00100000:1.0:1767320857.771360:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640512 00000100:00100000:1.0:1767320857.771374:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640512 00000100:00100000:2.0:1767320857.771453:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5471 00000100:00100000:2.0:1767320857.771465:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920640640:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320857.771551:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920640640:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 88us (1753us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320857.771558:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5471 00000100:00100000:2.0:1767320857.771575:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6233 00000100:00100000:2.0:1767320857.771579:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920640512:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320857.771600:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320857.771627:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920640512:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (663us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320857.771643:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6233 00000100:00080000:0.0:1767320857.771953:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320857.819322:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320857.819342:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320857.819346:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320857.819349:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320857.819354:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320857.819361:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6100 00000100:00080000:0.0:1767320857.819366:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320857.819373:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320857.819414:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63100 x1853163985532032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320857.819442:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320857.819445:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320857.819447:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320857.819451:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6100 00000100:00080000:0.0:1767320857.819453:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320857.819456:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320857.819470:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62300 x1853163985532160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320857.819539:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320857.819543:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320857.819545:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320857.819548:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6100 00000100:00080000:0.0:1767320857.819551:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320857.819554:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320857.819567:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61880 x1853163985532416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320857.819595:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320857.819599:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320857.819601:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320857.819604:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6100 00000100:00080000:0.0:1767320857.819606:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320857.819609:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320857.819620:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61500 x1853163985532672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320857.819792:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985532544:0@lo:400:kworker.0 00000100:00100000:0.0:1767320857.819846:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320857.819897:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320857.819901:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985532800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320857.819917:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320857.819944:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532800 00000100:00100000:0.0:1767320857.819951:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532800 00000100:00100000:0.0:1767320857.819992:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3549 00000100:00100000:0.0:1767320857.820001:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985532800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320857.820045:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985532800:12345-0@lo:400:kworker.0 Request processed in 46us (126us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320857.820050:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3549 00000100:00100000:0.0:1767320857.820082:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985532800:0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820160:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532544 00000100:00100000:3.0:1767320857.820187:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532544 00000100:00100000:3.0:1767320857.820215:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5472 00000100:00100000:3.0:1767320857.820219:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985532544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820245:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985532544:12345-0@lo:400:kworker.0 Request processed in 27us (399us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320857.820249:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5472 00000100:00100000:3.0:1767320857.820273:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985532544:0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820295:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985532928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820319:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320857.820349:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320857.820354:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985532288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820368:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320857.820387:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532288 00000100:00100000:3.0:1767320857.820389:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532288 00000100:00100000:3.0:1767320857.820405:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5473 00000100:00100000:3.0:1767320857.820409:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985532288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820430:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985532288:12345-0@lo:400:kworker.0 Request processed in 20us (60us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320857.820433:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5473 00000100:00100000:3.0:1767320857.820443:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532928 00000100:00100000:3.0:1767320857.820445:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532928 00000100:00100000:3.0:1767320857.820458:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3550 00000100:00100000:3.0:1767320857.820462:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985532928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820476:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985532928:12345-0@lo:400:kworker.0 Request processed in 14us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320857.820480:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3550 00000100:00100000:3.0:1767320857.820508:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985532288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.820522:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985532928:0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.821104:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985532032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320857.821143:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.821207:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985532160:0@lo:38:kworker.0 00000100:00100000:2.0:1767320857.821232:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.821271:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985532416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320857.821291:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.821307:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985532672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320857.821328:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.821356:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532032 00000100:00100000:2.0:1767320857.821361:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532032 00000100:00100000:2.0:1767320857.821379:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532416 00000100:00100000:2.0:1767320857.821382:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532416 00000100:00100000:2.0:1767320857.821394:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532672 00000100:00100000:2.0:1767320857.821397:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532672 00000100:00100000:2.0:1767320857.821409:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6234 00000100:00100000:2.0:1767320857.821492:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985532032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320857.821513:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320857.821559:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985532032:12345-0@lo:38:kworker.0 Request processed in 142us (413us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320857.821566:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6234 00000100:00100000:2.0:1767320857.821581:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6235 00000100:00100000:2.0:1767320857.821584:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985532416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320857.821593:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320857.821869:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985532416:12345-0@lo:38:kworker.0 Request processed in 281us (575us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320857.821875:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6235 00000100:00100000:2.0:1767320857.821989:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6236 00000100:00100000:2.0:1767320857.821995:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985532672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320857.822005:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320857.822035:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985532672:12345-0@lo:38:kworker.0 Request processed in 39us (706us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320857.822040:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6236 00000100:00080000:2.0:1767320857.822078:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320857.822083:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320857.822087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985532032:0@lo:38:kworker.0 00000100:00080000:2.0:1767320857.822095:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320857.822098:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320857.822101:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985532416:0@lo:38:kworker.0 00000100:00080000:0.0:1767320857.822103:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320857.822107:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320857.822110:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320857.822113:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985532672:0@lo:38:kworker.0 00000100:00100000:1.0:1767320857.822534:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985532160 00000100:00100000:1.0:1767320857.822540:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985532160 00000100:00100000:1.0:1767320857.822568:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2049 00000100:00100000:1.0:1767320857.822574:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985532160:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320857.822585:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320857.822638:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985532160:12345-0@lo:38:kworker.0 Request processed in 56us (1397us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320857.822645:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2049 00000100:00080000:0.0:1767320857.822804:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320857.822809:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320857.822813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985532160:0@lo:38:kworker.0 00010000:00010000:2.0:1767320858.459888:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320859.483404:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320860.507432:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320861.403517:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320861.403541:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985533056:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320861.403608:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320861.403868:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533056 00000100:00100000:2.0:1767320861.403877:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533056 00000100:00100000:2.0:1767320861.403937:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2760 00000100:00100000:2.0:1767320861.403946:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985533056:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320861.403975:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320861.404019:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985533056:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (410us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320861.404025:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2760 00000100:00100000:0.0:1767320861.404149:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985533056:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320861.404199:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320861.404347:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320861.404358:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985533184:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320861.404387:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320861.404799:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533184 00000100:00100000:3.0:1767320861.404805:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533184 00000100:00100000:3.0:1767320861.404855:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2761 00000100:00100000:3.0:1767320861.404862:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985533184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320861.404884:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320861.404925:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985533184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 63us (536us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320861.404932:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2761 00000100:00100000:3.0:1767320861.404965:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985533184:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320861.405026:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320861.532194:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320862.556447:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320862.893921:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320862.893982:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320862.894085:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640768 00000100:00100000:2.0:1767320862.894096:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640768 00000100:00100000:0.0:1767320862.894157:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920640896 00000100:00100000:2.0:1767320862.894196:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6237 00000100:00100000:2.0:1767320862.894208:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920640768:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:0.0:1767320862.894222:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920640896 00010000:00020000:2.0:1767320862.894239:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320862.894276:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5474 00000100:00100000:0.0:1767320862.894287:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920640896:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320862.894324:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920640768:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 115us (413us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320862.894333:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6237 00000100:00100000:0.0:1767320862.894339:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920640896:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 54us (355us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320862.894352:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5474 00000100:00080000:3.0:1767320862.894870:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320862.939353:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320862.939374:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320862.939377:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320862.939381:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320862.939386:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320862.939396:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6106 00000100:00080000:0.0:1767320862.939401:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320862.939418:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320862.939476:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63480 x1853163985533312/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320862.939506:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320862.939510:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320862.939512:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320862.939516:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6106 00000100:00080000:0.0:1767320862.939519:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320862.939523:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320862.939541:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63b80 x1853163985533440/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320862.939562:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985533312:0@lo:38:kworker.0 00000100:00080000:0.0:1767320862.939565:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320862.939571:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320862.939573:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320862.939577:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6106 00000100:00080000:0.0:1767320862.939580:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320862.939583:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320862.939596:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60a80 x1853163985533696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320862.939624:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320862.939627:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320862.939629:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320862.939634:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6106 00000100:00100000:2.0:1767320862.939636:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320862.939638:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320862.939644:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320862.939656:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61500 x1853163985533952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320862.939704:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985533824:0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.939732:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320862.939807:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985533568:0@lo:400:kworker.0 00000100:00100000:2.0:1767320862.939817:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985533440:0@lo:38:kworker.0 00000100:00100000:3.0:1767320862.939844:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320862.939856:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320862.939871:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320862.939879:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985534208:0@lo:400:kworker.0 00000100:00100000:2.0:1767320862.939896:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985533696:0@lo:38:kworker.0 00000100:00100000:3.0:1767320862.939905:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320862.939941:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985534208 00000100:00100000:3.0:1767320862.939948:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985534208 00000100:00100000:0.0:1767320862.940567:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533312 00000100:00100000:0.0:1767320862.940573:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533312 00000100:00100000:0.0:1767320862.940601:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533824 00000100:00100000:0.0:1767320862.940604:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533824 00000100:00100000:0.0:1767320862.940642:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533568 00000100:00100000:0.0:1767320862.940645:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533568 00000100:00100000:3.0:1767320862.940934:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3551 00000100:00100000:3.0:1767320862.940947:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985534208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320862.941010:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985534208:12345-0@lo:400:kworker.0 Request processed in 67us (1105us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320862.941018:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3551 00000100:00080000:3.0:1767320862.941047:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320862.941081:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985534208:0@lo:400:kworker.0 00000100:00100000:1.0:1767320862.941173:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985534080:0@lo:400:kworker.0 00000100:00100000:1.0:1767320862.941213:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320862.941249:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985534080 00000100:00100000:1.0:1767320862.941253:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985534080 00000100:00100000:1.0:1767320862.941299:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3552 00000100:00100000:1.0:1767320862.941306:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985534080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320862.941339:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985534080:12345-0@lo:400:kworker.0 Request processed in 34us (126us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320862.941345:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3552 00000100:00100000:1.0:1767320862.941380:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985534080:0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.942050:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5475 00000100:00100000:0.0:1767320862.942063:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985533824:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.942114:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985533824:12345-0@lo:400:kworker.0 Request processed in 55us (2381us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320862.942146:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5476 00000100:00100000:3.0:1767320862.942155:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985533568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320862.942204:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985533568:12345-0@lo:400:kworker.0 Request processed in 51us (2360us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320862.942211:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5476 00000100:00100000:3.0:1767320862.942232:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6238 00000100:00100000:3.0:1767320862.942237:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985533312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320862.942255:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320862.942278:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320862.942285:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985533312:12345-0@lo:38:kworker.0 Request processed in 46us (2651us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320862.942292:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6238 00000100:00100000:3.0:1767320862.942304:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533696 00000100:00100000:2.0:1767320862.942308:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985533952:0@lo:38:kworker.0 00000100:00100000:3.0:1767320862.942309:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533696 00000100:00100000:3.0:1767320862.942329:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6239 00000100:00100000:3.0:1767320862.942335:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985533696:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320862.942343:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767320862.942347:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320862.942377:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320862.942383:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320862.942387:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985533312:0@lo:38:kworker.0 00000100:00100000:2.0:1767320862.942429:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533952 00000100:00100000:2.0:1767320862.942433:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533952 00000100:00100000:2.0:1767320862.942451:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6240 00000100:00100000:2.0:1767320862.942455:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985533952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320862.942466:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320862.942498:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985533952:12345-0@lo:38:kworker.0 Request processed in 40us (155us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320862.942504:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6240 00000100:00100000:3.0:1767320862.942558:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985533696:12345-0@lo:38:kworker.0 Request processed in 221us (2631us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320862.942564:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6239 00000100:00100000:3.0:1767320862.942595:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985533568:0@lo:400:kworker.0 00000100:00100000:3.0:1767320862.942640:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985533440 00000100:00100000:3.0:1767320862.942644:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985533440 00000100:00100000:3.0:1767320862.942663:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2050 00000100:00100000:3.0:1767320862.942668:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985533440:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320862.942678:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320862.942701:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985533440:12345-0@lo:38:kworker.0 Request processed in 33us (2846us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320862.942707:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2050 00000100:00080000:3.0:1767320862.942737:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320862.942743:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320862.942746:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985533696:0@lo:38:kworker.0 00000100:00080000:3.0:1767320862.942756:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320862.942759:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320862.942762:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985533952:0@lo:38:kworker.0 00000100:00080000:3.0:1767320862.942931:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320862.942935:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320862.942939:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985533440:0@lo:38:kworker.0 00000100:00100000:0.0:1767320862.943118:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5475 00000100:00100000:2.0:1767320862.947120:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985533824:0@lo:400:kworker.0 00010000:00010000:1.0:1767320863.579937:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320864.605630:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320865.627654:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320866.523805:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320866.523830:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985534336:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320866.523903:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320866.524407:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320866.524419:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985534464:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320866.524454:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320866.526429:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985534336 00000100:00100000:0.0:1767320866.526445:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985534336 00000100:00100000:0.0:1767320866.526604:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985534464 00000100:00100000:0.0:1767320866.526609:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985534464 00000100:00100000:1.0:1767320866.526672:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2762 00000100:00100000:1.0:1767320866.526684:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985534336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320866.526728:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320866.526823:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985534336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (2920us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320866.526831:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2762 00000100:00100000:1.0:1767320866.526900:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985534336:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320866.529009:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320866.529018:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320866.529222:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2763 00000100:00100000:0.0:1767320866.529240:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985534464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320866.529283:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320866.530040:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985534464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 805us (5581us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320866.530049:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2763 00000100:00100000:1.0:1767320866.530222:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985534464:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320866.663441:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320867.035686:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320842/real 1767320842] req@ffff8ce19db74a80 x1853163985528064/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320867 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320867.035723:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19db74a80 x1853163985528064/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320867 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320867.035745:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320867.035774:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320867.035778:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985528064:0@lo:250:kworker.0 00000100:00080000:1.0:1767320867.035938:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320867.035944:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320867.035947:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320867.035952:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6090 00000100:00080000:1.0:1767320867.035957:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320867.035962:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320867.036009:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adf80 x1853163985534592/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320867.036035:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320867.036040:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320867.036042:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320867.036048:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6111 00000100:00080000:1.0:1767320867.036051:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320867.036055:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320867.036073:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985534720/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320867.036076:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985534592:0@lo:250:kworker.0 00000100:00080000:1.0:1767320867.036089:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320867.036092:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320867.036094:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320867.036097:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6111 00000100:00080000:1.0:1767320867.036099:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320867.036102:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320867.036113:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af100 x1853163985534848/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:0.0:1767320867.036129:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985534592 offset 0 length 520: 4 00000100:00080000:1.0:1767320867.036157:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320867.036160:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320867.036193:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320867.036197:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6111 00000100:00080000:1.0:1767320867.036199:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320867.036202:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320867.036204:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985534720:0@lo:38:kworker.0 00000100:00100000:1.0:1767320867.036214:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adc00 x1853163985535104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320867.036237:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320867.036242:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320867.036245:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320867.036247:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320867.036251:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6111 00000100:00080000:1.0:1767320867.036253:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320867.036255:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985534848:0@lo:38:kworker.0 00000100:00080000:1.0:1767320867.036257:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320867.036267:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bce00 x1853163985535360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320867.036280:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320867.036295:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985535104:0@lo:38:kworker.0 00000100:00100000:0.0:1767320867.036312:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320867.036324:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985535232:0@lo:400:kworker.0 00000100:00100000:0.0:1767320867.036327:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985535360:0@lo:38:kworker.0 00000100:00100000:0.0:1767320867.036342:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320867.036382:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985534720 00000100:00100000:1.0:1767320867.036390:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985534720 00000100:00100000:0.0:1767320867.036407:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985535104 00000100:00100000:0.0:1767320867.036410:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985535104 00000100:00100000:1.0:1767320867.036420:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985535360 00000100:00100000:1.0:1767320867.036422:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985535360 00000100:00100000:0.0:1767320867.036430:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6241 00000100:00100000:1.0:1767320867.036435:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6242 00000100:00100000:0.0:1767320867.036437:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985534720:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320867.036438:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985535104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320867.036455:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320867.036506:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985535104:12345-0@lo:38:kworker.0 Request processed in 65us (190us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320867.036513:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6242 00000100:00100000:2.0:1767320867.036524:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985534976:0@lo:400:kworker.0 00000100:00100000:1.0:1767320867.036530:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6243 00000100:00100000:1.0:1767320867.036535:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985535360:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320867.036545:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320867.036546:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320867.036575:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320867.036578:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985534720:12345-0@lo:38:kworker.0 Request processed in 140us (340us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320867.036578:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985535360:12345-0@lo:38:kworker.0 Request processed in 39us (232us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320867.036585:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6243 00000100:00100000:1.0:1767320867.036595:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985534976 00000100:00100000:1.0:1767320867.036599:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985534976 00000100:00100000:2.0:1767320867.036606:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320867.036615:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985535616:0@lo:400:kworker.0 00000100:00100000:2.0:1767320867.036651:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320867.036697:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985534848 00000100:00100000:2.0:1767320867.036702:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985534848 00000100:00100000:2.0:1767320867.036729:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2051 00000100:00100000:2.0:1767320867.036733:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190db8050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985534848:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320867.036745:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320867.036771:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190db8050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985534848:12345-0@lo:38:kworker.0 Request processed in 36us (489us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320867.036777:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2051 00000100:00100000:2.0:1767320867.036971:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320867.036994:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320867.037000:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985535488:0@lo:400:kworker.0 00000100:00100000:2.0:1767320867.037022:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320867.037049:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985535616 00000100:00100000:2.0:1767320867.037053:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985535616 00000100:00100000:2.0:1767320867.037087:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985535488 00000100:00100000:2.0:1767320867.037090:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985535488 00000100:00100000:2.0:1767320867.037112:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3553 00000100:00100000:2.0:1767320867.037119:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985535616:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320867.037195:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985535616:12345-0@lo:400:kworker.0 Request processed in 47us (517us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320867.037210:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3553 00000100:00100000:2.0:1767320867.037227:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3554 00000100:00100000:2.0:1767320867.037234:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985535488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320867.037274:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985535488:12345-0@lo:400:kworker.0 Request processed in 41us (250us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320867.037280:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3554 00000100:00100000:2.0:1767320867.037303:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985535232 00000100:00100000:2.0:1767320867.037307:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985535232 00000100:00080000:3.0:1767320867.038314:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320867.038396:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985535616:0@lo:400:kworker.0 00000100:00100000:3.0:1767320867.039110:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985535488:0@lo:400:kworker.0 00000100:00080000:3.0:1767320867.039153:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320867.039159:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320867.039188:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985535104:0@lo:38:kworker.0 00000100:00080000:3.0:1767320867.039201:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320867.039205:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320867.039208:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985535360:0@lo:38:kworker.0 00000100:00080000:3.0:1767320867.039218:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320867.039222:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320867.039225:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985534720:0@lo:38:kworker.0 00000100:00080000:3.0:1767320867.039236:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320867.039238:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320867.039241:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985534848:0@lo:38:kworker.0 00000100:00100000:0.0:1767320867.039297:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6241 00000100:00100000:2.0:1767320867.039414:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5477 00000100:00100000:2.0:1767320867.039431:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985534976:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320867.039521:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985534976:12345-0@lo:400:kworker.0 Request processed in 99us (2948us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320867.039529:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5477 00000100:00100000:2.0:1767320867.039546:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5478 00000100:00100000:2.0:1767320867.039552:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985535232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320867.039698:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985534976:0@lo:400:kworker.0 00000100:00100000:2.0:1767320867.039817:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985535232:12345-0@lo:400:kworker.0 Request processed in 264us (2845us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320867.039824:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5478 00000100:00100000:3.0:1767320867.039887:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985535232:0@lo:400:kworker.0 00010000:00010000:1.0:1767320867.678230:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320868.010677:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767320868.010718:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920641024 offset 0 length 520: 4 00000100:00100000:1.0:1767320868.010741:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320868.010812:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920641280 00000100:00100000:2.0:1767320868.010822:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920641280 00000100:00100000:2.0:1767320868.010888:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920641152 00000100:00100000:2.0:1767320868.010891:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920641152 00000100:00100000:2.0:1767320868.010918:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5479 00000100:00100000:2.0:1767320868.010929:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920641280:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320868.011006:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920641280:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (341us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320868.011014:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5479 00000100:00100000:2.0:1767320868.011033:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6244 00000100:00100000:2.0:1767320868.011037:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920641152:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320868.011057:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320868.011079:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920641152:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (338us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320868.011085:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6244 00000100:00080000:3.0:1767320868.011551:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320868.699478:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320869.723569:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320870.756101:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320871.646036:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985535744:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320871.646134:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320871.646191:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985535872:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320871.646204:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320871.646295:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320871.646375:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985535744 00000100:00100000:3.0:1767320871.646387:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985535744 00000100:00100000:2.0:1767320871.646458:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985535872 00000100:00100000:2.0:1767320871.646470:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985535872 00000100:00100000:3.0:1767320871.646504:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2764 00000100:00100000:3.0:1767320871.646519:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985535744:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320871.646562:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2765 00000100:00100000:2.0:1767320871.646580:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985535872:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320871.646584:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:2.0:1767320871.646627:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320871.646664:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985535744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 147us (498us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320871.646674:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2764 00000100:00080000:1.0:1767320871.646751:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320871.646758:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320871.646970:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985535744:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320871.659207:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985535872:12345-0@lo:13:osp-pre-0-1.0 Request processed in 12620us (12901us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320871.659229:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2765 00000100:00100000:0.0:1767320871.659414:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985535872:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320871.772242:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320872.155496:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320872.155511:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320872.155514:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320872.155519:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320872.155525:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320872.155535:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6115 00000100:00080000:1.0:1767320872.155552:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320872.155560:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320872.155616:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bc380 x1853163985536000/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320872.155650:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320872.155654:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320872.155657:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320872.155661:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6115 00000100:00080000:1.0:1767320872.155663:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320872.155668:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320872.155685:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bca80 x1853163985536128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320872.155749:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320872.155754:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320872.155756:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320872.155760:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6115 00000100:00080000:1.0:1767320872.155763:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320872.155766:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320872.155782:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bea00 x1853163985536384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320872.155822:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320872.155827:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320872.155829:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320872.155833:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6115 00000100:00080000:1.0:1767320872.155838:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320872.155843:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320872.155844:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320872.155860:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af1bf800 x1853163985536640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320872.155873:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985536256:0@lo:400:kworker.0 00000100:00100000:2.0:1767320872.155948:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320872.156008:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985536896:0@lo:400:kworker.0 00000100:00100000:2.0:1767320872.156027:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320872.156055:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536256 00000100:00100000:2.0:1767320872.156061:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536256 00000100:00100000:2.0:1767320872.156113:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5480 00000100:00100000:2.0:1767320872.156121:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985536256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320872.156158:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985536256:12345-0@lo:400:kworker.0 Request processed in 39us (209us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320872.156197:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5480 00000100:00100000:3.0:1767320872.156197:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985536768:0@lo:400:kworker.0 00000100:00080000:2.0:1767320872.156229:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320872.156273:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985536256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320872.156286:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320872.156333:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536896 00000100:00100000:2.0:1767320872.156340:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536896 00000100:00100000:3.0:1767320872.156359:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536768 00000100:00100000:3.0:1767320872.156367:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536768 00000100:00100000:2.0:1767320872.156408:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3555 00000100:00100000:2.0:1767320872.156420:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985536896:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320872.156445:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985536512:0@lo:400:kworker.0 00000100:00100000:2.0:1767320872.156474:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985536896:12345-0@lo:400:kworker.0 Request processed in 55us (442us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320872.156494:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.156583:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985536000:0@lo:38:kworker.0 00000100:00100000:1.0:1767320872.156615:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320872.156659:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536512 00000100:00100000:1.0:1767320872.156660:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985536128:0@lo:38:kworker.0 00000100:00100000:0.0:1767320872.156670:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536512 00000100:00100000:0.0:1767320872.156712:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536000 00000100:00100000:0.0:1767320872.156715:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536000 00000100:00100000:1.0:1767320872.156721:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320872.156735:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5481 00000100:00100000:0.0:1767320872.156743:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985536512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320872.156770:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985536384:0@lo:38:kworker.0 00000100:00100000:0.0:1767320872.156799:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985536512:12345-0@lo:400:kworker.0 Request processed in 55us (304us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320872.156805:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5481 00000100:00100000:1.0:1767320872.156817:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.157111:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af1bf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985536640:0@lo:38:kworker.0 00000100:00100000:0.0:1767320872.157128:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536384 00000100:00100000:0.0:1767320872.157134:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536384 00000100:00100000:1.0:1767320872.157142:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320872.157152:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536640 00000100:00100000:0.0:1767320872.157156:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536640 00000100:00100000:0.0:1767320872.157219:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6245 00000100:00100000:0.0:1767320872.157225:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985536000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320872.157260:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320872.157264:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6246 00000100:00100000:1.0:1767320872.157283:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985536384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320872.157303:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320872.157316:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985536000:12345-0@lo:38:kworker.0 Request processed in 88us (700us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320872.157324:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6245 00000100:00100000:0.0:1767320872.157341:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6247 00000100:00100000:0.0:1767320872.157346:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985536640:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320872.157355:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320872.157363:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985536384:12345-0@lo:38:kworker.0 Request processed in 78us (548us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320872.157372:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6246 00000100:00100000:0.0:1767320872.157808:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985536640:12345-0@lo:38:kworker.0 Request processed in 458us (665us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320872.157811:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985536512:0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.157820:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6247 00000100:00100000:0.0:1767320872.157883:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985536896:0@lo:400:kworker.0 00000100:00080000:1.0:1767320872.157887:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320872.157896:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320872.157902:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985536000:0@lo:38:kworker.0 00000100:00080000:1.0:1767320872.157917:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320872.157928:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320872.157936:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320872.157942:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985536384:0@lo:38:kworker.0 00000100:00080000:1.0:1767320872.157959:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320872.157963:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985536128 00000100:00080000:1.0:1767320872.157965:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320872.157969:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985536128 00000100:00100000:1.0:1767320872.157974:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985536640:0@lo:38:kworker.0 00000100:00100000:0.0:1767320872.158009:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2052 00000100:00100000:0.0:1767320872.158017:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985536128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320872.158026:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320872.158059:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1be300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985536128:12345-0@lo:38:kworker.0 Request processed in 42us (1341us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320872.158068:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2052 00000100:00080000:1.0:1767320872.158083:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320872.158110:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320872.158115:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985536128:0@lo:38:kworker.0 00000100:00100000:3.0:1767320872.162234:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3556 00000100:00100000:3.0:1767320872.162299:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985536768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320872.162483:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985536768:12345-0@lo:400:kworker.0 Request processed in 232us (6204us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320872.162499:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3556 00000100:00100000:0.0:1767320872.162618:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af1bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985536768:0@lo:400:kworker.0 00000100:00100000:2.0:1767320872.165228:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3555 00010000:00010000:0.0:1767320872.796395:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320873.130946:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320873.131004:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320873.132019:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920641408 00000100:00100000:1.0:1767320873.132043:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920641408 00000100:00100000:1.0:1767320873.132152:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920641536 00000100:00100000:1.0:1767320873.132156:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920641536 00000100:00100000:1.0:1767320873.132282:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5482 00000100:00100000:1.0:1767320873.132297:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920641536:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320873.132444:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920641536:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 150us (1439us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320873.132457:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5482 00000100:00100000:1.0:1767320873.132480:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6248 00000100:00100000:1.0:1767320873.132485:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920641408:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320873.132509:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320873.132541:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920641408:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (1609us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320873.132548:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6248 00000100:00080000:3.0:1767320873.135044:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320873.819778:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320874.843695:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320875.867620:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320876.764122:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985537024:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320876.764222:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320876.764349:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537024 00000100:00100000:2.0:1767320876.764357:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537024 00000100:00100000:2.0:1767320876.764412:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2766 00000100:00100000:2.0:1767320876.764428:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985537024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320876.764459:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320876.764502:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985537024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (280us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320876.764507:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2766 00000100:00100000:2.0:1767320876.764545:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985537024:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320876.764662:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320876.765809:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985537152:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320876.765857:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320876.765929:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537152 00000100:00100000:3.0:1767320876.765935:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537152 00000100:00100000:3.0:1767320876.765985:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2767 00000100:00100000:3.0:1767320876.765991:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985537152:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320876.766010:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320876.766043:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985537152:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (185us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320876.766048:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2767 00000100:00100000:3.0:1767320876.766078:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985537152:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320876.766123:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320876.892325:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320877.276097:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320877.276109:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320877.276112:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320877.276116:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320877.276122:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320877.276132:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6120 00000100:00080000:0.0:1767320877.276137:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320877.276146:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320877.276230:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61500 x1853163985537280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320877.276265:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320877.276270:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320877.276272:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320877.276275:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6120 00000100:00080000:0.0:1767320877.276277:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320877.276280:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320877.276292:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61c00 x1853163985537408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320877.276337:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320877.276340:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320877.276342:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320877.276344:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6120 00000100:00080000:0.0:1767320877.276346:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320877.276349:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320877.276356:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61880 x1853163985537664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320877.276376:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320877.276379:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320877.276380:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320877.276383:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6120 00000100:00080000:0.0:1767320877.276385:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320877.276387:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320877.276396:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74e00 x1853163985537920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320877.276424:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985537792:0@lo:400:kworker.0 00000100:00100000:0.0:1767320877.276491:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320877.276603:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985537280:0@lo:38:kworker.0 00000100:00100000:1.0:1767320877.276626:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320877.276652:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985537408:0@lo:38:kworker.0 00000100:00100000:1.0:1767320877.276667:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320877.276679:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985537664:0@lo:38:kworker.0 00000100:00100000:1.0:1767320877.276691:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320877.276711:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537792 00000100:00100000:1.0:1767320877.276717:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537792 00000100:00100000:1.0:1767320877.276762:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537280 00000100:00100000:1.0:1767320877.276764:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537280 00000100:00100000:1.0:1767320877.276775:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537664 00000100:00100000:1.0:1767320877.276777:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537664 00000100:00100000:1.0:1767320877.276788:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5483 00000100:00100000:1.0:1767320877.276795:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985537792:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320877.276835:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985537792:12345-0@lo:400:kworker.0 Request processed in 41us (343us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320877.276840:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5483 00000100:00100000:1.0:1767320877.276854:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6249 00000100:00100000:1.0:1767320877.276856:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985537280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320877.276876:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320877.276895:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985537280:12345-0@lo:38:kworker.0 Request processed in 38us (268us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320877.276900:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6249 00000100:00100000:1.0:1767320877.276905:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6250 00000100:00100000:1.0:1767320877.276908:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985537664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320877.276913:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320877.276923:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985537664:12345-0@lo:38:kworker.0 Request processed in 14us (232us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320877.276926:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6250 00000100:00080000:1.0:1767320877.276936:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320877.276963:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985537792:0@lo:400:kworker.0 00000100:00080000:1.0:1767320877.276990:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320877.276995:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320877.276998:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985537664:0@lo:38:kworker.0 00000100:00100000:1.0:1767320877.277003:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985537920:0@lo:38:kworker.0 00000100:00100000:1.0:1767320877.277024:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320877.277044:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537920 00000100:00100000:1.0:1767320877.277047:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537920 00000100:00100000:1.0:1767320877.277066:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6251 00000100:00100000:1.0:1767320877.277070:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985537920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320877.277079:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320877.277104:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985537920:12345-0@lo:38:kworker.0 Request processed in 32us (79us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320877.277110:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6251 00000100:00080000:1.0:1767320877.277131:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320877.277135:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320877.277138:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985537920:0@lo:38:kworker.0 00000100:00080000:1.0:1767320877.277190:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320877.277194:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320877.277197:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985537280:0@lo:38:kworker.0 00000100:00100000:2.0:1767320877.277985:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320877.278005:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985537536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320877.278050:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320877.278111:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537536 00000100:00100000:2.0:1767320877.278116:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537536 00000100:00100000:2.0:1767320877.278213:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5484 00000100:00100000:2.0:1767320877.278221:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985537536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320877.278280:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985537536:12345-0@lo:400:kworker.0 Request processed in 61us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320877.278287:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5484 00000100:00080000:2.0:1767320877.278306:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320877.278331:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985537536:0@lo:400:kworker.0 00000100:00100000:0.0:1767320877.278426:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985537408 00000100:00100000:0.0:1767320877.278431:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985537408 00000100:00100000:0.0:1767320877.278466:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2053 00000100:00100000:0.0:1767320877.278471:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985537408:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320877.278489:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320877.278529:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985537408:12345-0@lo:38:kworker.0 Request processed in 56us (1859us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320877.278536:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2053 00000100:00080000:1.0:1767320877.278866:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320877.278873:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320877.278878:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985537408:0@lo:38:kworker.0 00010000:00010000:2.0:1767320877.916457:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320878.258529:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320878.258603:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920641664 00000100:00100000:2.0:1767320878.258616:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920641664 00000100:00100000:1.0:1767320878.258636:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320878.258735:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920641792 00000100:00100000:2.0:1767320878.258739:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920641792 00000100:00080000:1.0:1767320878.258815:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320878.259195:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5485 00000100:00100000:0.0:1767320878.259208:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920641792:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320878.259241:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6252 00000100:00100000:3.0:1767320878.259248:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920641664:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320878.259267:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320878.259314:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920641792:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (677us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320878.259322:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5485 00000100:00100000:3.0:1767320878.259772:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920641664:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 520us (1248us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320878.259781:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6252 00010000:00010000:0.0:1767320878.940757:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320879.965790:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320881.000807:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320881.884046:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320881.884095:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985538048:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320881.884195:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320881.884324:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538048 00000100:00100000:3.0:1767320881.884333:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538048 00000100:00100000:3.0:1767320881.884394:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2768 00000100:00100000:3.0:1767320881.884403:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985538048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320881.884433:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320881.884481:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985538048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (285us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320881.884487:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2768 00000100:00100000:3.0:1767320881.884788:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320881.884797:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985538176:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320881.884823:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320881.884844:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538176 00000100:00100000:3.0:1767320881.884848:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538176 00000100:00100000:3.0:1767320881.884873:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2769 00000100:00100000:3.0:1767320881.884879:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985538176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320881.884894:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320881.884922:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985538176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (98us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320881.884928:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2769 00000100:00100000:3.0:1767320881.884968:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985538176:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320881.885180:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320881.885184:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320881.885210:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985538048:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320882.012889:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320882.395830:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320882.395843:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320882.395845:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320882.395850:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320882.395855:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320882.395865:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6125 00000100:00080000:3.0:1767320882.395871:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320882.395878:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320882.395919:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938a80 x1853163985538304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320882.396113:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985538304:0@lo:38:kworker.0 00000100:00100000:1.0:1767320882.396217:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320882.396347:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538304 00000100:00100000:2.0:1767320882.396356:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538304 00000100:00100000:2.0:1767320882.396408:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6253 00000100:00100000:2.0:1767320882.396417:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985538304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320882.396440:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320882.396509:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320882.396523:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320882.396526:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320882.396529:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6125 00000100:00080000:3.0:1767320882.396532:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320882.396536:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320882.396553:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a300 x1853163985538432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320882.400216:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985538432:0@lo:38:kworker.0 00000100:00100000:1.0:1767320882.400296:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320882.400435:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538432 00000100:00100000:0.0:1767320882.400444:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538432 00000100:00100000:0.0:1767320882.400481:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2054 00000100:00100000:0.0:1767320882.400489:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985538432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320882.400512:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320882.400554:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985538432:12345-0@lo:38:kworker.0 Request processed in 64us (257us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320882.400561:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2054 00000100:00100000:2.0:1767320882.400748:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985538304:12345-0@lo:38:kworker.0 Request processed in 4330us (4530us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320882.400755:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6253 00000100:00080000:1.0:1767320882.401116:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320882.401123:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320882.401126:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985538304:0@lo:38:kworker.0 00000100:00080000:1.0:1767320882.401151:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320882.401155:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320882.401158:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985538432:0@lo:38:kworker.0 00000100:00100000:0.0:1767320882.402158:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985538560:0@lo:400:kworker.0 00000100:00100000:0.0:1767320882.402223:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320882.402313:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538560 00000100:00100000:2.0:1767320882.402320:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538560 00000100:00100000:2.0:1767320882.402389:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5486 00000100:00100000:2.0:1767320882.402398:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985538560:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320882.402480:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320882.402902:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320882.402908:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320882.402912:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320882.402917:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6125 00000100:00080000:3.0:1767320882.402921:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320882.402926:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320882.402949:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a300 x1853163985538688/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320882.402983:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320882.402987:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320882.402988:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320882.402992:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6125 00000100:00080000:3.0:1767320882.402995:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320882.402998:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320882.403010:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6680 x1853163985538944/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320882.403042:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985538816:0@lo:400:kworker.0 00000100:00100000:3.0:1767320882.403084:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320882.403124:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538816 00000100:00100000:3.0:1767320882.403128:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538816 00000100:00100000:3.0:1767320882.403180:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5487 00000100:00100000:3.0:1767320882.403188:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985538816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320882.403224:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985538816:12345-0@lo:400:kworker.0 Request processed in 37us (138us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320882.403231:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5487 00000100:00100000:3.0:1767320882.403273:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985538816:0@lo:400:kworker.0 00000100:00080000:3.0:1767320882.403289:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320882.403299:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985538688:0@lo:38:kworker.0 00000100:00100000:3.0:1767320882.403321:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320882.403335:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985538944:0@lo:38:kworker.0 00000100:00100000:3.0:1767320882.403354:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320882.403378:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538688 00000100:00100000:3.0:1767320882.403382:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538688 00000100:00100000:3.0:1767320882.403400:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985538944 00000100:00100000:3.0:1767320882.403403:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985538944 00000100:00100000:3.0:1767320882.403419:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6254 00000100:00100000:3.0:1767320882.403424:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985538688:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320882.403443:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320882.403474:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985538688:12345-0@lo:38:kworker.0 Request processed in 48us (152us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320882.403481:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6254 00000100:00100000:3.0:1767320882.403493:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6255 00000100:00100000:3.0:1767320882.403497:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985538944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320882.403506:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320882.403529:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985538944:12345-0@lo:38:kworker.0 Request processed in 30us (173us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320882.403535:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6255 00000100:00080000:3.0:1767320882.403562:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320882.403566:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320882.403569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985538688:0@lo:38:kworker.0 00000100:00080000:3.0:1767320882.403576:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320882.403580:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320882.403583:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985538944:0@lo:38:kworker.0 00000100:00100000:2.0:1767320882.403871:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985538560:12345-0@lo:400:kworker.0 Request processed in 1473us (1646us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320882.403878:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5486 00000100:00100000:0.0:1767320882.403970:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985538560:0@lo:400:kworker.0 00010000:00010000:2.0:1767320883.039471:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320883.371302:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320883.371393:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920641920 00000100:00100000:2.0:1767320883.371404:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920641920 00000100:00100000:2.0:1767320883.371459:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6256 00000100:00100000:2.0:1767320883.371467:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920641920:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320883.371491:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320883.371564:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920641920:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 96us (272us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320883.371571:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6256 00000100:00100000:1.0:1767320883.377991:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320883.378143:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920642048 00000100:00100000:0.0:1767320883.378153:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920642048 00000100:00100000:0.0:1767320883.378262:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5488 00000100:00100000:0.0:1767320883.378272:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920642048:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320883.378364:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920642048:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (384us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320883.378372:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5488 00000100:00080000:3.0:1767320883.378886:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320884.059532:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320885.089125:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320886.108978:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320887.004017:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320887.004048:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985539200:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320887.004130:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320887.004298:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539200 00000100:00100000:2.0:1767320887.004306:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539200 00000100:00100000:2.0:1767320887.004380:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2770 00000100:00100000:2.0:1767320887.004392:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985539200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320887.004624:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320887.004684:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985539200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 295us (553us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320887.004693:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2770 00000100:00100000:2.0:1767320887.004748:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985539200:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320887.005597:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985539072:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320887.005653:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320887.005718:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320887.006073:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539072 00000100:00100000:3.0:1767320887.006081:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539072 00000100:00100000:3.0:1767320887.006138:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2771 00000100:00100000:3.0:1767320887.006147:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985539072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320887.006193:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320887.006253:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985539072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 107us (599us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320887.006260:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2771 00000100:00100000:0.0:1767320887.006358:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985539072:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320887.006539:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320887.132227:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320887.517525:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320887.517537:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320887.517542:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320887.517545:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320887.517551:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320887.517559:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6130 00000100:00080000:3.0:1767320887.517564:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320887.517571:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320887.517612:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985539328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320887.517644:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320887.517648:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320887.517650:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320887.517654:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6130 00000100:00080000:3.0:1767320887.517657:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320887.517662:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320887.517682:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5f80 x1853163985539456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320887.517746:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320887.517751:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320887.517752:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320887.517756:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6130 00000100:00080000:3.0:1767320887.517758:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320887.517761:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320887.517773:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6300 x1853163985539712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320887.517798:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320887.517801:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320887.517804:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320887.517807:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6130 00000100:00080000:3.0:1767320887.517809:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320887.517812:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320887.517824:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6680 x1853163985539968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320887.517854:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320887.517880:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985539840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320887.518005:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320887.518477:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985539584:0@lo:400:kworker.0 00000100:00100000:1.0:1767320887.518532:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320887.518746:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539840 00000100:00100000:2.0:1767320887.518754:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539840 00000100:00100000:2.0:1767320887.518815:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539584 00000100:00100000:2.0:1767320887.518818:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539584 00000100:00100000:2.0:1767320887.518845:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5489 00000100:00100000:2.0:1767320887.518861:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985539840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320887.518950:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985539840:12345-0@lo:400:kworker.0 Request processed in 96us (945us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320887.518959:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5489 00000100:00100000:2.0:1767320887.518993:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5490 00000100:00100000:2.0:1767320887.518999:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985539584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320887.519031:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985539584:12345-0@lo:400:kworker.0 Request processed in 32us (500us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320887.519037:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5490 00000100:00100000:2.0:1767320887.519090:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985539584:0@lo:400:kworker.0 00000100:00100000:2.0:1767320887.519148:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985539840:0@lo:400:kworker.0 00000100:00100000:0.0:1767320887.519456:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985539328:0@lo:38:kworker.0 00000100:00100000:0.0:1767320887.519491:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320887.519551:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985539456:0@lo:38:kworker.0 00000100:00100000:0.0:1767320887.519599:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320887.519642:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985539712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320887.519675:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320887.519690:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985539968:0@lo:38:kworker.0 00000100:00100000:0.0:1767320887.519711:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320887.519740:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539456 00000100:00100000:0.0:1767320887.519745:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539456 00000100:00100000:0.0:1767320887.519769:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2055 00000100:00100000:0.0:1767320887.519774:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985539456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320887.519797:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320887.519833:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985539456:12345-0@lo:38:kworker.0 Request processed in 59us (235us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320887.519840:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2055 00000100:00080000:0.0:1767320887.519872:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320887.519876:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320887.519879:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985539456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320887.519942:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539328 00000100:00100000:2.0:1767320887.519946:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539328 00000100:00080000:1.0:1767320887.520080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320887.520352:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539712 00000100:00100000:2.0:1767320887.520362:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539712 00000100:00100000:0.0:1767320887.520427:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985539968 00000100:00100000:0.0:1767320887.520431:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985539968 00000100:00100000:0.0:1767320887.520449:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6257 00000100:00100000:0.0:1767320887.520454:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985539328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320887.520468:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320887.520522:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985539328:12345-0@lo:38:kworker.0 Request processed in 64us (1028us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320887.520533:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6257 00000100:00100000:0.0:1767320887.520561:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6258 00000100:00100000:0.0:1767320887.520568:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985539712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320887.520583:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320887.520621:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985539712:12345-0@lo:38:kworker.0 Request processed in 51us (944us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320887.520630:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6258 00000100:00100000:0.0:1767320887.520649:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6259 00000100:00100000:0.0:1767320887.520656:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985539968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320887.520671:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320887.520709:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985539968:12345-0@lo:38:kworker.0 Request processed in 51us (994us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320887.520719:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6259 00000100:00080000:0.0:1767320887.520757:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320887.520762:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320887.520765:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985539328:0@lo:38:kworker.0 00000100:00080000:0.0:1767320887.520773:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320887.520775:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320887.520778:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985539712:0@lo:38:kworker.0 00000100:00080000:0.0:1767320887.520785:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320887.520788:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320887.520790:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985539968:0@lo:38:kworker.0 00010000:00010000:1.0:1767320888.159648:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320888.489658:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320888.489799:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920642304 00000100:00100000:0.0:1767320888.489808:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920642304 00000100:00100000:0.0:1767320888.489881:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5491 00000100:00100000:0.0:1767320888.489891:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920642304:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320888.489998:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920642304:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 108us (348us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320888.490007:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5491 00000100:00100000:3.0:1767320888.490557:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320888.490622:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920642176 00000100:00100000:0.0:1767320888.490629:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920642176 00000100:00100000:0.0:1767320888.490663:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6260 00000100:00100000:0.0:1767320888.490669:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920642176:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320888.490690:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320888.490731:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920642176:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 61us (176us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320888.490737:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6260 00000100:00080000:1.0:1767320888.490924:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320889.180053:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320890.204465:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320891.231215:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320892.124054:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320892.124076:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985540096:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320892.124154:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320892.124298:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985540096 00000100:00100000:2.0:1767320892.124307:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985540096 00000100:00100000:2.0:1767320892.124398:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2772 00000100:00100000:2.0:1767320892.124410:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985540096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320892.124453:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320892.124510:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985540096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (355us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320892.124518:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2772 00000100:00100000:2.0:1767320892.124568:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985540096:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320892.124641:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320892.125308:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320892.125333:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985540224:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320892.125388:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.125524:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985540224 00000100:00100000:3.0:1767320892.125531:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985540224 00000100:00100000:3.0:1767320892.125585:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2773 00000100:00100000:3.0:1767320892.125595:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985540224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320892.125622:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320892.125664:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985540224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (275us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320892.125673:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2773 00000100:00080000:1.0:1767320892.125732:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320892.125762:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985540224:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320892.251581:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767320892.591744:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920642432 offset 0 length 520: 4 00000100:00100000:2.0:1767320892.591792:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320892.591980:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320892.596670:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920642688 00000100:00100000:0.0:1767320892.596687:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920642688 00000100:00100000:0.0:1767320892.596882:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920642560 00000100:00100000:0.0:1767320892.596886:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920642560 00000100:00100000:0.0:1767320892.596911:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5492 00000100:00100000:0.0:1767320892.596921:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920642688:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320892.597017:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920642688:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 96us (5223us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320892.597025:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5492 00000100:00100000:0.0:1767320892.597048:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6261 00000100:00100000:0.0:1767320892.597053:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920642560:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320892.597076:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320892.597102:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920642560:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (5122us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320892.597107:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6261 00000100:00080000:1.0:1767320892.602577:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320892.635313:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320867/real 1767320867] req@ffff8ce1886adf80 x1853163985534592/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320892 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320892.635348:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1886adf80 x1853163985534592/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320892 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320892.635367:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320892.635399:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320892.635403:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985534592:0@lo:250:kworker.0 00000100:00080000:1.0:1767320892.635646:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320892.635651:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320892.635654:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320892.635658:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6115 00000100:00080000:1.0:1767320892.635662:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320892.635668:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320892.635702:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985540352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320892.635727:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320892.635731:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320892.635733:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320892.635736:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6135 00000100:00080000:1.0:1767320892.635738:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320892.635743:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320892.635759:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519880 x1853163985540480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320892.635768:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320892.635771:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320892.635773:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320892.635777:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6135 00000100:00080000:1.0:1767320892.635779:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320892.635782:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320892.635793:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b800 x1853163985540608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320892.635827:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320892.635831:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320892.635833:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320892.635836:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6135 00000100:00080000:1.0:1767320892.635838:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320892.635842:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320892.635854:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3518700 x1853163985540864/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320892.635890:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320892.635894:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320892.635896:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320892.635899:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6135 00000100:00080000:1.0:1767320892.635901:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320892.635905:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320892.635915:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519f80 x1853163985541120/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320892.635971:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320892.635982:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985540992:0@lo:400:kworker.0 00000100:00100000:1.0:1767320892.636043:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.636269:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985540736:0@lo:400:kworker.0 00000100:00100000:3.0:1767320892.636297:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.636334:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985540992 00000100:00100000:3.0:1767320892.636342:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985540992 00000100:00100000:2.0:1767320892.636471:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985540736 00000100:00100000:2.0:1767320892.636476:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985540736 00000100:00100000:3.0:1767320892.638620:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5493 00000100:00100000:3.0:1767320892.638636:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985540992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320892.638732:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985540992:12345-0@lo:400:kworker.0 Request processed in 100us (2689us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320892.638739:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5493 00000100:00100000:3.0:1767320892.638791:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985540992:0@lo:400:kworker.0 00000100:00100000:3.0:1767320892.638848:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985540352:0@lo:250:kworker.0 00000400:00000100:3.0:1767320892.638888:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985540352 offset 0 length 520: 4 00000100:00100000:3.0:1767320892.638916:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985540480:0@lo:38:kworker.0 00000100:00100000:3.0:1767320892.638942:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.638960:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985540608:0@lo:38:kworker.0 00000100:00100000:3.0:1767320892.638981:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.638998:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985540864:0@lo:38:kworker.0 00000100:00100000:3.0:1767320892.639015:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.639029:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985541120:0@lo:38:kworker.0 00000100:00100000:3.0:1767320892.639047:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.639072:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985540480 00000100:00100000:3.0:1767320892.639077:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985540480 00000100:00100000:3.0:1767320892.639092:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985540864 00000100:00100000:3.0:1767320892.639094:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985540864 00000100:00100000:3.0:1767320892.639101:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985541120 00000100:00100000:3.0:1767320892.639103:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985541120 00000100:00100000:3.0:1767320892.639111:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6262 00000100:00100000:3.0:1767320892.639115:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985540480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320892.639133:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320892.640415:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985540480:12345-0@lo:38:kworker.0 Request processed in 1291us (1465us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320892.640427:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6262 00000100:00100000:3.0:1767320892.640458:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6263 00000100:00100000:3.0:1767320892.640464:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985540864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320892.640481:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320892.640527:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985540864:12345-0@lo:38:kworker.0 Request processed in 63us (1510us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320892.640533:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6263 00000100:00100000:3.0:1767320892.640543:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6264 00000100:00100000:3.0:1767320892.640548:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985541120:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320892.640558:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320892.640579:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985541120:12345-0@lo:38:kworker.0 Request processed in 30us (1531us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320892.640585:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6264 00000100:00080000:3.0:1767320892.640633:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320892.640641:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320892.640645:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985540480:0@lo:38:kworker.0 00000100:00080000:3.0:1767320892.640655:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320892.640658:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320892.640662:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985540864:0@lo:38:kworker.0 00000100:00080000:3.0:1767320892.640670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320892.640672:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320892.640675:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985541120:0@lo:38:kworker.0 00000100:00100000:2.0:1767320892.641535:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5494 00000100:00100000:2.0:1767320892.641545:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985540736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320892.641611:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985540736:12345-0@lo:400:kworker.0 Request processed in 68us (5313us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320892.641618:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5494 00000100:00100000:2.0:1767320892.641663:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985540736:0@lo:400:kworker.0 00000100:00080000:2.0:1767320892.641698:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320892.643178:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985540608 00000100:00100000:0.0:1767320892.643200:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985540608 00000100:00100000:0.0:1767320892.643239:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2056 00000100:00100000:0.0:1767320892.643247:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985540608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320892.643271:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320892.643348:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985540608:12345-0@lo:38:kworker.0 Request processed in 101us (4364us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320892.643356:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2056 00000100:00080000:0.0:1767320892.643427:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320892.643433:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320892.643438:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985540608:0@lo:38:kworker.0 00010000:00010000:2.0:1767320893.275921:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320894.301140:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320895.323490:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320896.350220:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320897.243395:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320897.243409:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320897.243411:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985541248:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320897.243417:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985541376:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320897.243574:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320897.243655:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985541376 00000100:00100000:3.0:1767320897.243663:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985541376 00000100:00100000:3.0:1767320897.243727:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2774 00000100:00100000:3.0:1767320897.243738:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985541376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320897.243855:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320897.243910:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985541376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 174us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320897.243918:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2774 00000100:00080000:2.0:1767320897.244002:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320897.244041:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985541376:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320897.244465:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320897.244579:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985541248 00000100:00100000:3.0:1767320897.244585:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985541248 00000100:00100000:3.0:1767320897.244663:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2775 00000100:00100000:3.0:1767320897.244672:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985541248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320897.244690:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320897.244728:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985541248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 57us (262us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320897.244735:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2775 00000100:00100000:0.0:1767320897.244826:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985541248:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320897.245496:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767320897.371895:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320897.706306:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320897.706414:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920642944 00000100:00100000:0.0:1767320897.706426:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920642944 00000100:00100000:0.0:1767320897.706509:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5495 00000100:00100000:0.0:1767320897.706520:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920642944:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320897.706609:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920642944:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (313us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320897.706629:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5495 00000100:00100000:3.0:1767320897.707221:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320897.707286:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920642816 00000100:00100000:1.0:1767320897.707299:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920642816 00000100:00100000:1.0:1767320897.707333:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6265 00000100:00100000:1.0:1767320897.707339:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920642816:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320897.707362:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320897.707404:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920642816:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 64us (186us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320897.707411:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6265 00000100:00080000:2.0:1767320897.708201:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320897.757647:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320897.757661:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320897.757665:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320897.757669:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320897.757674:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320897.757683:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6140 00000100:00080000:0.0:1767320897.757688:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320897.757697:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320897.757743:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9180 x1853163985541504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320897.757787:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320897.757791:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320897.757793:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320897.757796:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6140 00000100:00080000:0.0:1767320897.757800:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320897.757804:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320897.757821:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb100 x1853163985541632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320897.757924:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320897.758130:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985541504:0@lo:38:kworker.0 00000100:00100000:3.0:1767320897.758222:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320897.758283:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985541632:0@lo:38:kworker.0 00000100:00100000:3.0:1767320897.758310:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320897.758358:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985541632 00000100:00100000:3.0:1767320897.758366:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985541632 00000100:00100000:3.0:1767320897.758408:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2057 00000100:00100000:3.0:1767320897.758415:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985541632:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320897.758440:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320897.758481:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985541632:12345-0@lo:38:kworker.0 Request processed in 64us (169us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320897.758488:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2057 00000100:00080000:3.0:1767320897.758533:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320897.758538:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320897.758542:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985541632:0@lo:38:kworker.0 00000100:00080000:0.0:1767320897.758793:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320897.758798:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320897.758801:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320897.758808:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6140 00000100:00080000:0.0:1767320897.758811:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320897.758816:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320897.758836:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb800 x1853163985541888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320897.758874:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320897.758877:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320897.758879:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320897.758883:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6140 00000100:00080000:0.0:1767320897.758885:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320897.758888:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320897.758898:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9f80 x1853163985542144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320897.758957:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985541504 00000100:00100000:2.0:1767320897.758963:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985541504 00000100:00100000:2.0:1767320897.758991:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6266 00000100:00100000:2.0:1767320897.758995:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985541504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320897.759006:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320897.759038:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985541504:12345-0@lo:38:kworker.0 Request processed in 41us (817us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320897.759045:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6266 00000100:00100000:2.0:1767320897.759084:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985542016:0@lo:400:kworker.0 00000100:00100000:2.0:1767320897.759111:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320897.759132:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320897.759137:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985542272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320897.759159:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320897.759201:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542272 00000100:00100000:2.0:1767320897.759205:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542272 00000100:00100000:2.0:1767320897.759238:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3557 00000100:00100000:2.0:1767320897.759244:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985542272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320897.759289:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985542272:12345-0@lo:400:kworker.0 Request processed in 45us (128us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320897.759296:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3557 00000100:00100000:2.0:1767320897.759313:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542016 00000100:00100000:2.0:1767320897.759317:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542016 00000100:00100000:2.0:1767320897.759349:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5496 00000100:00100000:2.0:1767320897.759354:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985542016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320897.759381:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985542016:12345-0@lo:400:kworker.0 Request processed in 28us (270us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320897.759387:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5496 00000100:00080000:2.0:1767320897.759401:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320897.759424:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985542272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320897.759433:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985542016:0@lo:400:kworker.0 00000100:00100000:3.0:1767320897.759512:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985541888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320897.759539:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320897.759569:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985542144:0@lo:38:kworker.0 00000100:00100000:3.0:1767320897.759588:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320897.759606:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320897.759610:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320897.759624:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985541504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320897.759828:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985541888 00000100:00100000:2.0:1767320897.759832:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985541888 00000100:00100000:2.0:1767320897.759861:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542144 00000100:00100000:2.0:1767320897.759864:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542144 00000100:00100000:2.0:1767320897.759881:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6267 00000100:00100000:2.0:1767320897.759886:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985541888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320897.759898:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320897.759929:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985541888:12345-0@lo:38:kworker.0 Request processed in 42us (389us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320897.759934:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6267 00000100:00100000:2.0:1767320897.759942:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6268 00000100:00100000:2.0:1767320897.759945:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985542144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320897.759951:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320897.759966:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985542144:12345-0@lo:38:kworker.0 Request processed in 20us (378us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320897.759969:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6268 00000100:00080000:3.0:1767320897.760033:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320897.760038:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320897.760041:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985541888:0@lo:38:kworker.0 00000100:00080000:3.0:1767320897.760050:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320897.760053:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320897.760055:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985542144:0@lo:38:kworker.0 00000100:00100000:1.0:1767320897.760139:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985542400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.760171:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320897.760190:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985541760:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.760203:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320897.760259:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985541760 00000100:00100000:2.0:1767320897.760263:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985541760 00000100:00100000:2.0:1767320897.760298:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5497 00000100:00100000:2.0:1767320897.760304:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985541760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320897.760337:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985541760:12345-0@lo:400:kworker.0 Request processed in 33us (132us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320897.760343:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5497 00000100:00100000:3.0:1767320897.760389:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542400 00000100:00100000:3.0:1767320897.760392:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542400 00000100:00100000:3.0:1767320897.760416:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3558 00000100:00100000:3.0:1767320897.760420:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985542400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320897.760443:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985542400:12345-0@lo:400:kworker.0 Request processed in 23us (281us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320897.760446:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3558 00000100:00080000:1.0:1767320897.760501:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320897.760516:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985542400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.760525:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985541760:0@lo:400:kworker.0 00010000:00010000:1.0:1767320898.395398:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320899.422754:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320900.443489:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320901.467648:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320902.363687:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320902.363735:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985542528:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320902.363801:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320902.364148:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320902.364158:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985542656:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320902.364204:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320902.364276:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542528 00000100:00100000:2.0:1767320902.364293:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542528 00000100:00100000:2.0:1767320902.364345:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542656 00000100:00100000:2.0:1767320902.364348:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542656 00000100:00100000:2.0:1767320902.364370:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2776 00000100:00100000:2.0:1767320902.364380:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985542528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320902.364410:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320902.364451:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985542528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (649us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320902.364458:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2776 00000100:00100000:2.0:1767320902.364475:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2777 00000100:00100000:2.0:1767320902.364481:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985542656:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320902.364496:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320902.364523:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985542656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (318us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320902.364532:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2777 00000100:00100000:2.0:1767320902.364574:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985542656:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320902.364602:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985542528:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320902.364734:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320902.364738:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320902.491446:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320902.825997:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320902.826231:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920643200 00000100:00100000:0.0:1767320902.826244:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920643200 00000100:00100000:0.0:1767320902.826327:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5498 00000100:00100000:0.0:1767320902.826337:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920643200:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320902.826418:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920643200:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (429us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320902.826426:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5498 00000100:00080000:1.0:1767320902.826827:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320902.827012:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320902.827067:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920643072 00000100:00100000:3.0:1767320902.827072:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920643072 00000100:00100000:3.0:1767320902.827104:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6269 00000100:00100000:3.0:1767320902.827110:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920643072:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320902.827131:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320902.827197:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920643072:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 85us (185us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320902.827203:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6269 00000100:00080000:2.0:1767320902.877897:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320902.877907:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320902.877911:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320902.877915:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320902.877919:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320902.877926:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6146 00000100:00080000:2.0:1767320902.877931:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320902.877937:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320902.877976:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9c000 x1853163985542784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320902.877992:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320902.877995:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320902.877997:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320902.878000:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6146 00000100:00080000:2.0:1767320902.878003:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320902.878006:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320902.878018:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9e680 x1853163985542912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320902.878058:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320902.878063:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320902.878064:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320902.878068:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6146 00000100:00080000:2.0:1767320902.878070:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320902.878073:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320902.878085:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6300 x1853163985543168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320902.878113:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320902.878116:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320902.878118:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320902.878121:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6146 00000100:00080000:2.0:1767320902.878123:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320902.878126:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320902.878135:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5f80 x1853163985543424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320902.878190:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985543296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.878253:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320902.878314:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320902.878324:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985543552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.878349:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320902.878384:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543552 00000100:00100000:2.0:1767320902.878392:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543552 00000100:00100000:2.0:1767320902.878440:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3559 00000100:00100000:2.0:1767320902.878449:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985543552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.878501:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985543552:12345-0@lo:400:kworker.0 Request processed in 53us (150us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320902.878509:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3559 00000100:00100000:2.0:1767320902.878553:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985543552:0@lo:400:kworker.0 00000100:00100000:0.0:1767320902.878633:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985543680:0@lo:400:kworker.0 00000100:00100000:0.0:1767320902.878661:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320902.878694:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320902.878700:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985543040:0@lo:400:kworker.0 00000100:00100000:0.0:1767320902.878722:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320902.878748:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543296 00000100:00100000:0.0:1767320902.878752:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543296 00000100:00100000:0.0:1767320902.878784:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543040 00000100:00100000:0.0:1767320902.878788:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543040 00000100:00100000:0.0:1767320902.878805:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5499 00000100:00100000:0.0:1767320902.878811:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985543296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320902.878842:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985543296:12345-0@lo:400:kworker.0 Request processed in 32us (590us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320902.878848:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5499 00000100:00100000:0.0:1767320902.878861:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5500 00000100:00100000:0.0:1767320902.878867:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985543040:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320902.878896:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985543040:12345-0@lo:400:kworker.0 Request processed in 30us (174us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320902.878902:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5500 00000100:00100000:0.0:1767320902.878925:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985543040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320902.879009:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985542784:0@lo:38:kworker.0 00000100:00100000:3.0:1767320902.879037:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320902.879069:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985542912:0@lo:38:kworker.0 00000100:00100000:3.0:1767320902.879092:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320902.879115:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985543168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320902.879134:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320902.879150:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985543424:0@lo:38:kworker.0 00000100:00100000:3.0:1767320902.879192:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320902.879220:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542912 00000100:00100000:3.0:1767320902.879224:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542912 00000100:00100000:3.0:1767320902.879246:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2058 00000100:00100000:3.0:1767320902.879250:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985542912:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320902.879268:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320902.879301:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985542912:12345-0@lo:38:kworker.0 Request processed in 50us (208us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320902.879307:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2058 00000100:00100000:3.0:1767320902.879325:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985542784 00000100:00100000:3.0:1767320902.879329:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985542784 00000100:00100000:3.0:1767320902.879340:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543168 00000100:00100000:3.0:1767320902.879342:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543168 00000100:00100000:3.0:1767320902.879349:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543424 00000100:00100000:3.0:1767320902.879351:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543424 00000100:00100000:3.0:1767320902.879360:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6270 00000100:00100000:3.0:1767320902.879364:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985542784:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320902.879374:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320902.879398:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985542784:12345-0@lo:38:kworker.0 Request processed in 32us (359us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320902.879403:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6270 00000100:00100000:3.0:1767320902.879413:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6271 00000100:00100000:3.0:1767320902.879416:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985543168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320902.879425:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320902.879445:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985543168:12345-0@lo:38:kworker.0 Request processed in 27us (310us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320902.879451:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6271 00000100:00100000:3.0:1767320902.879461:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6272 00000100:00100000:3.0:1767320902.879465:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985543424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320902.879473:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320902.879493:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985543424:12345-0@lo:38:kworker.0 Request processed in 27us (301us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320902.879499:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6272 00000100:00080000:3.0:1767320902.879530:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320902.879535:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320902.879538:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985542784:0@lo:38:kworker.0 00000100:00080000:3.0:1767320902.879548:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320902.879551:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320902.879554:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985542912:0@lo:38:kworker.0 00000100:00080000:3.0:1767320902.879560:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320902.879562:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320902.879566:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985543168:0@lo:38:kworker.0 00000100:00080000:3.0:1767320902.879572:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320902.879575:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320902.879577:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985543424:0@lo:38:kworker.0 00000100:00080000:1.0:1767320902.879704:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320902.879773:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543680 00000100:00100000:2.0:1767320902.879778:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543680 00000100:00100000:2.0:1767320902.879816:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3560 00000100:00100000:2.0:1767320902.879823:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985543680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.879869:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985543680:12345-0@lo:400:kworker.0 Request processed in 47us (1207us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320902.879875:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3560 00000100:00100000:2.0:1767320902.879903:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985543296:0@lo:400:kworker.0 00000100:00100000:3.0:1767320902.880003:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985543680:0@lo:400:kworker.0 00010000:00010000:0.0:1767320903.519016:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320904.539429:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320905.563945:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320906.588019:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320907.483723:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985543808:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320907.483851:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320907.483934:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543808 00000100:00100000:2.0:1767320907.483947:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543808 00000100:00080000:1.0:1767320907.484038:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320907.484044:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2778 00000100:00100000:2.0:1767320907.484058:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985543808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320907.484115:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320907.484224:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985543808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 170us (373us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320907.484232:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2778 00000100:00100000:1.0:1767320907.484494:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985543808:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320907.485350:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320907.485370:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985543936:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320907.485431:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320907.485551:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985543936 00000100:00100000:3.0:1767320907.485560:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985543936 00000100:00100000:3.0:1767320907.485646:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2779 00000100:00100000:3.0:1767320907.485659:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985543936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320907.485690:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320907.485741:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985543936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (309us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320907.485749:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2779 00000100:00100000:3.0:1767320907.485811:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985543936:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320907.486085:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320907.614266:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320907.946705:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320907.946873:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920643456 00000100:00100000:2.0:1767320907.946888:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920643456 00000100:00080000:3.0:1767320907.946971:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320907.947006:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5501 00000100:00100000:2.0:1767320907.947022:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920643456:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320907.947134:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920643456:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 117us (446us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320907.947144:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5501 00000100:00100000:1.0:1767320907.953062:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320907.956217:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920643328 00000100:00100000:0.0:1767320907.956241:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920643328 00000100:00100000:0.0:1767320907.956328:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6273 00000100:00100000:0.0:1767320907.956339:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920643328:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320907.956369:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320907.956477:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920643328:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 137us (3428us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320907.956486:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6273 00000100:00080000:2.0:1767320907.997931:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320907.997944:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320907.997948:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320907.997952:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320907.997958:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320907.997970:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6151 00000100:00080000:2.0:1767320907.997976:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320907.997986:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320907.998044:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af800 x1853163985544064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320907.998074:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320907.998078:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320907.998080:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320907.998083:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6151 00000100:00080000:2.0:1767320907.998086:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320907.998089:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320907.998103:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886adc00 x1853163985544192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320907.998154:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320907.998158:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320907.998160:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320907.998184:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6151 00000100:00080000:2.0:1767320907.998187:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320907.998191:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320907.998203:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ace00 x1853163985544448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320907.998235:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320907.998239:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320907.998240:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320907.998244:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6151 00000100:00080000:2.0:1767320907.998247:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320907.998251:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320907.998264:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4a80 x1853163985544704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320907.998315:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985544832:0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.998405:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320907.998502:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320907.998509:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985544576:0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.998541:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320907.998599:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544832 00000100:00100000:2.0:1767320907.998609:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544832 00000100:00100000:3.0:1767320907.998662:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985544320:0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.998691:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3561 00000100:00100000:3.0:1767320907.998703:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320907.998707:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985544832:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320907.998736:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320907.998743:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985544960:0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.998781:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985544832:12345-0@lo:400:kworker.0 Request processed in 79us (375us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320907.998791:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3561 00000100:00100000:2.0:1767320907.998829:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544576 00000100:00100000:2.0:1767320907.998834:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544576 00000100:00100000:2.0:1767320907.998876:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544320 00000100:00100000:2.0:1767320907.998879:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544320 00000100:00100000:2.0:1767320907.998902:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5502 00000100:00100000:2.0:1767320907.998908:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985544576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.998943:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985544576:12345-0@lo:400:kworker.0 Request processed in 35us (402us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320907.998950:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5502 00000100:00100000:2.0:1767320907.998961:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5503 00000100:00100000:2.0:1767320907.998967:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985544320:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.998992:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985544320:12345-0@lo:400:kworker.0 Request processed in 25us (291us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320907.998998:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5503 00000100:00100000:2.0:1767320907.999049:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985544576:0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.999061:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985544832:0@lo:400:kworker.0 00000100:00100000:3.0:1767320907.999074:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320907.999112:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985544320:0@lo:400:kworker.0 00000100:00100000:3.0:1767320907.999141:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544960 00000100:00100000:3.0:1767320907.999146:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544960 00000100:00080000:2.0:1767320907.999152:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320907.999405:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985544064:0@lo:38:kworker.0 00000100:00100000:0.0:1767320907.999442:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320907.999467:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985544192:0@lo:38:kworker.0 00000100:00100000:0.0:1767320907.999495:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320907.999530:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985544448:0@lo:38:kworker.0 00000100:00100000:0.0:1767320907.999552:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320907.999567:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985544704:0@lo:38:kworker.0 00000100:00100000:0.0:1767320907.999596:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320907.999636:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544192 00000100:00100000:0.0:1767320907.999640:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544192 00000100:00100000:0.0:1767320907.999670:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2059 00000100:00100000:0.0:1767320907.999675:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985544192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320907.999700:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320907.999738:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985544192:12345-0@lo:38:kworker.0 Request processed in 62us (242us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320907.999745:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2059 00000100:00100000:0.0:1767320907.999769:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544064 00000100:00100000:0.0:1767320907.999772:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544064 00000100:00100000:0.0:1767320907.999792:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544448 00000100:00100000:0.0:1767320907.999795:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544448 00000100:00100000:0.0:1767320907.999810:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985544704 00000100:00100000:0.0:1767320907.999813:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985544704 00000100:00100000:0.0:1767320907.999822:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6274 00000100:00100000:0.0:1767320907.999826:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985544064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320907.999835:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320907.999863:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985544064:12345-0@lo:38:kworker.0 Request processed in 35us (420us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320907.999871:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6274 00000100:00100000:0.0:1767320907.999882:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6275 00000100:00100000:0.0:1767320907.999885:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985544448:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320907.999893:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320907.999913:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985544448:12345-0@lo:38:kworker.0 Request processed in 26us (360us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320907.999917:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6275 00000100:00100000:0.0:1767320907.999927:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6276 00000100:00100000:0.0:1767320907.999931:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985544704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320907.999939:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320907.999957:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985544704:12345-0@lo:38:kworker.0 Request processed in 25us (371us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320907.999964:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6276 00000100:00080000:0.0:1767320908.000007:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320908.000012:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320908.000015:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985544064:0@lo:38:kworker.0 00000100:00080000:0.0:1767320908.000023:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320908.000026:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320908.000029:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985544192:0@lo:38:kworker.0 00000100:00080000:0.0:1767320908.000036:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320908.000039:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320908.000041:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985544448:0@lo:38:kworker.0 00000100:00080000:0.0:1767320908.000048:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320908.000051:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320908.000054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985544704:0@lo:38:kworker.0 00000100:00100000:3.0:1767320908.000800:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3562 00000100:00100000:3.0:1767320908.000816:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985544960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320908.000894:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985544960:12345-0@lo:400:kworker.0 Request processed in 86us (1822us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320908.000902:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3562 00000100:00100000:1.0:1767320908.000991:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985544960:0@lo:400:kworker.0 00010000:00010000:2.0:1767320908.635554:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320909.659525:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320910.683421:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320911.708352:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320912.603474:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320912.603515:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985545088:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320912.603590:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320912.603950:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320912.603963:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985545216:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320912.603996:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320912.604550:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545088 00000100:00100000:0.0:1767320912.604561:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545088 00000100:00100000:3.0:1767320912.604708:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545216 00000100:00100000:3.0:1767320912.604713:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545216 00000100:00100000:3.0:1767320912.604791:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2780 00000100:00100000:3.0:1767320912.604802:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985545216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320912.604843:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767320912.604913:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985545216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (914us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320912.604921:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2780 00000100:00100000:1.0:1767320912.605035:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985545216:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320912.605251:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320912.605520:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2781 00000100:00100000:0.0:1767320912.605530:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985545088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320912.605553:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320912.605620:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985545088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (2030us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320912.605627:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2781 00000100:00100000:3.0:1767320912.605754:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985545088:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320912.606621:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320912.731451:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320913.068312:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320913.068417:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320913.068580:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920643712 00000100:00100000:2.0:1767320913.068596:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920643712 00000100:00100000:2.0:1767320913.068697:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920643584 00000100:00100000:2.0:1767320913.068701:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920643584 00000100:00100000:2.0:1767320913.068729:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5504 00000100:00100000:2.0:1767320913.068746:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920643712:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320913.068869:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920643712:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 125us (572us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320913.068878:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5504 00000100:00100000:1.0:1767320913.068925:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6277 00000100:00100000:1.0:1767320913.068941:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920643584:12345-192.168.204.26@tcp:38:kworker.0 00000100:00080000:3.0:1767320913.068954:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:1.0:1767320913.068969:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320913.069050:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920643584:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 110us (631us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320913.073876:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6277 00000100:00080000:0.0:1767320913.115453:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320913.115464:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320913.115467:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320913.115471:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320913.115476:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320913.115487:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6156 00000100:00080000:0.0:1767320913.115504:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320913.115514:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320913.115562:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9180 x1853163985545344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320913.115594:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320913.115598:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320913.115600:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320913.115604:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6156 00000100:00080000:0.0:1767320913.115607:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320913.115610:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320913.115623:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8380 x1853163985545472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320913.115667:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320913.115671:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320913.115673:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320913.115677:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6156 00000100:00080000:0.0:1767320913.115680:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320913.115683:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320913.115697:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb100 x1853163985545728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320913.115724:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1767320913.115738:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320913.115743:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320913.115745:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320913.115749:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6156 00000100:00080000:0.0:1767320913.115752:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320913.115756:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985545600:0@lo:400:kworker.0 00000100:00080000:0.0:1767320913.115760:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320913.115784:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1180 x1853163985545984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320913.115856:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985545856:0@lo:400:kworker.0 00000100:00100000:3.0:1767320913.115893:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320913.115958:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985545344:0@lo:38:kworker.0 00000100:00100000:2.0:1767320913.116054:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320913.116069:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985546240:0@lo:400:kworker.0 00000100:00100000:0.0:1767320913.117416:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320913.117523:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320913.117535:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985546112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320913.117579:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320913.117621:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985546112 00000100:00100000:0.0:1767320913.117630:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985546112 00000100:00100000:0.0:1767320913.117680:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3563 00000100:00100000:0.0:1767320913.117689:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985546112:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320913.117745:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985546112:12345-0@lo:400:kworker.0 Request processed in 57us (165us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320913.117752:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3563 00000100:00100000:0.0:1767320913.117794:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985546112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320913.117842:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545600 00000100:00100000:0.0:1767320913.117846:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545600 00000100:00100000:0.0:1767320913.117882:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545856 00000100:00100000:0.0:1767320913.117885:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545856 00000100:00100000:0.0:1767320913.117907:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5505 00000100:00100000:0.0:1767320913.117913:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985545600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320913.117944:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985545600:12345-0@lo:400:kworker.0 Request processed in 32us (2056us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320913.117950:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5505 00000100:00100000:0.0:1767320913.117962:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5506 00000100:00100000:0.0:1767320913.117968:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985545856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320913.117998:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985545856:12345-0@lo:400:kworker.0 Request processed in 30us (592us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320913.118004:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5506 00000100:00080000:0.0:1767320913.118027:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320913.118045:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985545856:0@lo:400:kworker.0 00000100:00100000:1.0:1767320913.121035:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320913.121048:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985545600:0@lo:400:kworker.0 00000100:00100000:1.0:1767320913.121189:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985545472:0@lo:38:kworker.0 00000100:00100000:1.0:1767320913.121231:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320913.121269:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985545728:0@lo:38:kworker.0 00000100:00100000:0.0:1767320913.121356:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545344 00000100:00100000:0.0:1767320913.121364:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545344 00000100:00100000:0.0:1767320913.121406:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6278 00000100:00100000:0.0:1767320913.121414:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985545344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320913.121435:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320913.121591:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545472 00000100:00100000:3.0:1767320913.121596:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545472 00000100:00100000:3.0:1767320913.121623:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2060 00000100:00100000:3.0:1767320913.121628:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985545472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320913.121637:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320913.121676:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985545472:12345-0@lo:38:kworker.0 Request processed in 47us (444us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320913.121683:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2060 00000100:00100000:1.0:1767320913.122059:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320913.122117:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545728 00000100:00100000:3.0:1767320913.122122:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545728 00000100:00100000:3.0:1767320913.122144:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6279 00000100:00100000:3.0:1767320913.122149:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985545728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320913.122159:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320913.123052:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320913.123134:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985546240 00000100:00100000:2.0:1767320913.123144:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985546240 00000100:00100000:2.0:1767320913.123216:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3564 00000100:00100000:2.0:1767320913.123226:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985546240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320913.123328:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985545984:0@lo:38:kworker.0 00000100:00100000:1.0:1767320913.123382:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320913.123450:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320913.123458:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320913.123463:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985545472:0@lo:38:kworker.0 00000100:00100000:2.0:1767320913.123514:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985546240:12345-0@lo:400:kworker.0 Request processed in 287us (466us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320913.123524:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985545984 00000100:00100000:2.0:1767320913.123532:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3564 00000100:00100000:1.0:1767320913.123536:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985545984 00000100:00100000:1.0:1767320913.123584:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6280 00000100:00100000:1.0:1767320913.123593:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985545984:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320913.123602:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985546240:0@lo:400:kworker.0 00010000:00020000:1.0:1767320913.123613:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320913.123666:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985545984:12345-0@lo:38:kworker.0 Request processed in 72us (281us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320913.123673:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6280 00000100:00080000:1.0:1767320913.123716:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320913.123721:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320913.123724:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985545984:0@lo:38:kworker.0 00000100:00100000:3.0:1767320913.124245:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985545728:12345-0@lo:38:kworker.0 Request processed in 2093us (2183us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320913.124253:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6279 00000100:00080000:1.0:1767320913.124326:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320913.124330:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320913.124334:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985545728:0@lo:38:kworker.0 00000100:00100000:0.0:1767320913.124455:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985545344:12345-0@lo:38:kworker.0 Request processed in 3039us (3430us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320913.124462:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6278 00000100:00080000:1.0:1767320913.124662:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320913.124667:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320913.124670:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985545344:0@lo:38:kworker.0 00010000:00010000:2.0:1767320913.757150:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320914.787430:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320915.803401:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320916.827512:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320917.211313:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320892/real 1767320892] req@ffff8ce1b351a300 x1853163985540352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320917 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320917.211361:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b351a300 x1853163985540352/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320917 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320917.211379:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320917.211409:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320917.211413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985540352:0@lo:250:kworker.0 00000100:00080000:0.0:1767320917.211668:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320917.211677:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320917.211681:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320917.211688:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6140 00000100:00080000:0.0:1767320917.211695:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320917.211705:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320917.211757:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1500 x1853163985546368/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320917.211792:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320917.211796:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320917.211798:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320917.211802:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6161 00000100:00080000:0.0:1767320917.211805:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320917.211809:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320917.211827:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0a80 x1853163985546496/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320917.211837:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320917.211842:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320917.211843:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320917.211847:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6161 00000100:00080000:0.0:1767320917.211849:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320917.211852:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320917.211864:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1880 x1853163985546624/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320917.211952:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1767320917.212717:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320917.212724:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320917.212726:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320917.212735:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6161 00000100:00080000:0.0:1767320917.212740:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320917.212745:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320917.212773:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985546880/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320917.212828:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320917.212832:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320917.212834:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320917.212839:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6161 00000100:00080000:0.0:1767320917.212841:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320917.212845:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320917.212859:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1f80 x1853163985547136/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320917.212950:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985546368:0@lo:250:kworker.0 00000100:00100000:2.0:1767320917.212952:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985547008:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.216856:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985547392:0@lo:400:kworker.0 00000400:00000100:1.0:1767320917.216991:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985546368 offset 0 length 520: 4 00000100:00100000:1.0:1767320917.217072:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985546496:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.217117:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.217156:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985546624:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.217205:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.217228:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985546880:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.217252:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.217275:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985547136:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.217296:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.217355:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985546496 00000100:00100000:1.0:1767320917.217367:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985546496 00000100:00100000:1.0:1767320917.217428:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985546880 00000100:00100000:1.0:1767320917.217431:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985546880 00000100:00100000:1.0:1767320917.217449:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547136 00000100:00100000:1.0:1767320917.217452:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547136 00000100:00100000:1.0:1767320917.217473:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6281 00000100:00100000:1.0:1767320917.217481:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985546496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320917.217522:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320917.217573:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985546496:12345-0@lo:38:kworker.0 Request processed in 90us (454us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320917.217581:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6281 00000100:00100000:1.0:1767320917.217600:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6282 00000100:00100000:1.0:1767320917.217604:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985546880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320917.217613:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320917.217637:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985546880:12345-0@lo:38:kworker.0 Request processed in 32us (384us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320917.217643:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6282 00000100:00100000:1.0:1767320917.217654:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6283 00000100:00100000:1.0:1767320917.217658:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985547136:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320917.217667:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320917.217688:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985547136:12345-0@lo:38:kworker.0 Request processed in 28us (391us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320917.217693:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6283 00000100:00080000:1.0:1767320917.217743:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320917.217748:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320917.217753:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985546496:0@lo:38:kworker.0 00000100:00080000:1.0:1767320917.217763:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320917.217766:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320917.217769:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985546880:0@lo:38:kworker.0 00000100:00080000:1.0:1767320917.217777:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320917.217781:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320917.217784:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985547136:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.217842:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985547264:0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.217872:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.217910:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547264 00000100:00100000:1.0:1767320917.217914:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547264 00000100:00100000:1.0:1767320917.217966:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3565 00000100:00100000:1.0:1767320917.217974:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985547264:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.218020:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985547264:12345-0@lo:400:kworker.0 Request processed in 48us (148us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320917.218026:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3565 00000100:00100000:1.0:1767320917.218059:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985547264:0@lo:400:kworker.0 00000100:00080000:1.0:1767320917.218095:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320917.218132:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985546624 00000100:00100000:1.0:1767320917.218136:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985546624 00000100:00100000:1.0:1767320917.218158:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2061 00000100:00100000:1.0:1767320917.218188:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985546624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320917.218200:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320917.218229:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985546624:12345-0@lo:38:kworker.0 Request processed in 65us (1023us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320917.218236:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2061 00000100:00080000:1.0:1767320917.218270:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320917.218274:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320917.218277:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985546624:0@lo:38:kworker.0 00000100:00100000:2.0:1767320917.219147:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.225732:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547008 00000100:00100000:1.0:1767320917.225755:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547008 00000100:00100000:1.0:1767320917.225872:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5507 00000100:00100000:1.0:1767320917.225889:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985547008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.226022:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985547008:12345-0@lo:400:kworker.0 Request processed in 139us (6879us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320917.226031:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5507 00000100:00080000:2.0:1767320917.226119:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320917.226207:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985547008:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.226549:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320917.226602:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985546752:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.226642:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320917.226773:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985546752 00000100:00100000:2.0:1767320917.226780:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985546752 00000100:00100000:1.0:1767320917.226783:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547392 00000100:00100000:1.0:1767320917.226790:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547392 00000100:00100000:1.0:1767320917.226833:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3566 00000100:00100000:2.0:1767320917.226838:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5508 00000100:00100000:1.0:1767320917.226843:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985547392:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320917.226851:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985546752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.226899:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985547392:12345-0@lo:400:kworker.0 Request processed in 58us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320917.226906:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985546752:12345-0@lo:400:kworker.0 Request processed in 58us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320917.226912:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3566 00000100:00100000:2.0:1767320917.226918:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5508 00000100:00080000:3.0:1767320917.227103:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320917.227108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320917.227134:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985547392:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.227146:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985546752:0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.723567:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320917.723603:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985547648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320917.723718:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985547520:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320917.723736:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320917.723798:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320917.724086:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547648 00000100:00100000:0.0:1767320917.724108:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547648 00000100:00100000:0.0:1767320917.724208:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547520 00000100:00100000:0.0:1767320917.724212:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547520 00000100:00100000:0.0:1767320917.724241:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2782 00000100:00100000:0.0:1767320917.724253:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985547648:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320917.724292:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320917.724348:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985547648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (613us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320917.724357:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2782 00000100:00100000:0.0:1767320917.724379:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2783 00000100:00100000:0.0:1767320917.724385:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985547520:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320917.724403:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320917.724450:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985547520:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (664us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320917.724457:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2783 00000100:00100000:0.0:1767320917.724519:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985547520:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320917.725110:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985547648:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320917.857039:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320918.187066:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767320918.187117:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920643840 offset 0 length 520: 4 00000100:00100000:1.0:1767320918.187152:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320918.187294:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644096 00000100:00100000:2.0:1767320918.187309:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644096 00000100:00100000:2.0:1767320918.187415:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920643968 00000100:00100000:2.0:1767320918.187419:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920643968 00000100:00100000:2.0:1767320918.187441:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5509 00000100:00100000:2.0:1767320918.187454:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920644096:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320918.187569:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920644096:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 117us (526us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320918.187578:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5509 00000100:00100000:2.0:1767320918.187602:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6284 00000100:00100000:2.0:1767320918.187606:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920643968:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320918.187633:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320918.187661:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920643968:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (509us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320918.187669:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6284 00000100:00080000:0.0:1767320918.187752:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320918.875705:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320919.899422:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320920.923465:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320921.949570:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320922.331382:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320922.331407:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320922.331411:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320922.331417:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320922.331424:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320922.331436:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6165 00000100:00080000:1.0:1767320922.331444:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320922.331454:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320922.331515:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89880 x1853163985547776/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320922.331618:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985547776:0@lo:38:kworker.0 00000100:00100000:0.0:1767320922.331709:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320922.331931:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320922.331936:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320922.331938:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320922.331942:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6165 00000100:00080000:1.0:1767320922.331945:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320922.331950:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320922.331969:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552d180 x1853163985547904/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320922.332292:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547776 00000100:00100000:3.0:1767320922.332303:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547776 00000100:00100000:3.0:1767320922.332613:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6285 00000100:00100000:3.0:1767320922.332624:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985547776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320922.332651:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320922.332747:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985547904:0@lo:38:kworker.0 00000100:00100000:0.0:1767320922.332787:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320922.332906:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320922.332910:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320922.332912:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320922.332917:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6165 00000100:00080000:1.0:1767320922.332920:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320922.332924:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320922.332941:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1180 x1853163985548160/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320922.332971:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320922.332974:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320922.332976:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320922.332980:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6165 00000100:00080000:1.0:1767320922.332983:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320922.332986:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320922.332998:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3480 x1853163985548416/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320922.333048:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985547904 00000100:00100000:1.0:1767320922.333053:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985547904 00000100:00100000:1.0:1767320922.333080:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2062 00000100:00100000:1.0:1767320922.333085:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985547904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320922.333098:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320922.333134:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985547904:12345-0@lo:38:kworker.0 Request processed in 47us (347us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320922.333143:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2062 00000100:00100000:1.0:1767320922.333192:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985548032:0@lo:400:kworker.0 00000100:00100000:1.0:1767320922.333221:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320922.333244:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320922.333249:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985548672:0@lo:400:kworker.0 00000100:00100000:1.0:1767320922.333286:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320922.333317:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548032 00000100:00100000:1.0:1767320922.333333:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548032 00000100:00100000:2.0:1767320922.333783:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985548544:0@lo:400:kworker.0 00000100:00100000:2.0:1767320922.333822:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320922.333848:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320922.333854:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985548288:0@lo:400:kworker.0 00000100:00100000:2.0:1767320922.333876:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320922.333910:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548288 00000100:00100000:2.0:1767320922.333915:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548288 00000100:00100000:2.0:1767320922.333949:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5510 00000100:00100000:2.0:1767320922.333956:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985548288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320922.334007:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985548288:12345-0@lo:400:kworker.0 Request processed in 51us (130us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320922.334014:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5510 00000100:00100000:2.0:1767320922.334062:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985548288:0@lo:400:kworker.0 00000100:00100000:2.0:1767320922.334102:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548672 00000100:00100000:2.0:1767320922.334106:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548672 00000100:00100000:2.0:1767320922.334136:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548544 00000100:00100000:2.0:1767320922.334139:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548544 00000100:00100000:2.0:1767320922.334159:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3567 00000100:00100000:2.0:1767320922.334189:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985548672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320922.334228:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985548672:12345-0@lo:400:kworker.0 Request processed in 62us (941us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320922.334234:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3567 00000100:00100000:2.0:1767320922.334246:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3568 00000100:00100000:2.0:1767320922.334252:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985548544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320922.334415:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985547776:12345-0@lo:38:kworker.0 Request processed in 1791us (2705us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320922.334424:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6285 00000100:00100000:3.0:1767320922.334495:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985548672:0@lo:400:kworker.0 00000100:00080000:3.0:1767320922.334670:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320922.334676:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320922.334678:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320922.335049:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985548544:12345-0@lo:400:kworker.0 Request processed in 796us (1225us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320922.335060:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3568 00000100:00080000:0.0:1767320922.335188:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320922.335194:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320922.335198:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985547776:0@lo:38:kworker.0 00000100:00100000:0.0:1767320922.335223:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985548160:0@lo:38:kworker.0 00000100:00100000:0.0:1767320922.335273:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320922.335295:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985548416:0@lo:38:kworker.0 00000100:00100000:0.0:1767320922.335353:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320922.335380:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320922.335535:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320922.335539:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985547904:0@lo:38:kworker.0 00000100:00100000:0.0:1767320922.335570:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548160 00000100:00100000:0.0:1767320922.335577:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548160 00000100:00100000:0.0:1767320922.335610:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548416 00000100:00100000:0.0:1767320922.335612:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548416 00000100:00100000:0.0:1767320922.335635:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6286 00000100:00100000:0.0:1767320922.335642:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985548160:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320922.335664:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320922.335701:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985548160:12345-0@lo:38:kworker.0 Request processed in 58us (438us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320922.335708:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6286 00000100:00100000:0.0:1767320922.335722:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6287 00000100:00100000:0.0:1767320922.335726:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985548416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320922.335735:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320922.335758:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985548416:12345-0@lo:38:kworker.0 Request processed in 30us (404us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320922.335764:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6287 00000100:00080000:0.0:1767320922.335788:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320922.335793:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320922.335796:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985548160:0@lo:38:kworker.0 00000100:00080000:0.0:1767320922.335807:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320922.335810:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320922.335813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985548416:0@lo:38:kworker.0 00000100:00100000:1.0:1767320922.336070:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5511 00000100:00100000:1.0:1767320922.336084:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985548032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320922.336149:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985548032:12345-0@lo:400:kworker.0 Request processed in 71us (2927us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320922.336156:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5511 00000100:00100000:0.0:1767320922.336297:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985548032:0@lo:400:kworker.0 00000100:00100000:3.0:1767320922.336600:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985548544:0@lo:400:kworker.0 00000100:00100000:2.0:1767320922.843632:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1767320922.843669:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985548800:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320922.843779:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320922.843843:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985548928:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320922.843869:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320922.843970:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548800 00000100:00100000:1.0:1767320922.843981:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548800 00000100:00100000:1.0:1767320922.844058:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985548928 00000100:00100000:1.0:1767320922.844062:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985548928 00000100:00100000:1.0:1767320922.844095:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2784 00000100:00100000:1.0:1767320922.844106:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985548800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320922.844153:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320922.844248:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985548800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 143us (469us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320922.844258:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2784 00000100:00100000:1.0:1767320922.844279:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2785 00000100:00100000:1.0:1767320922.844285:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985548928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320922.844303:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320922.844365:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985548928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (494us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320922.844375:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2785 00000100:00100000:2.0:1767320922.844804:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985548800:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320922.844822:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985548928:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320922.972607:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320923.309392:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320923.309459:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320923.313309:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644352 00000100:00100000:1.0:1767320923.313338:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644352 00000100:00100000:1.0:1767320923.313672:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644224 00000100:00100000:1.0:1767320923.313677:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644224 00000100:00100000:1.0:1767320923.313711:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5512 00000100:00100000:1.0:1767320923.313725:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920644352:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320923.313844:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920644352:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 122us (4470us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320923.313852:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5512 00000100:00100000:1.0:1767320923.314073:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6288 00000100:00100000:1.0:1767320923.314080:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920644224:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320923.314107:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320923.314135:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920644224:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (4676us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320923.314141:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6288 00000100:00080000:0.0:1767320923.315358:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320923.995736:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320925.020224:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320926.043601:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320927.067760:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320927.452579:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320927.452597:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320927.452602:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320927.452607:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320927.452613:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320927.452624:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6170 00000100:00080000:1.0:1767320927.452630:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320927.452640:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320927.452704:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9f80 x1853163985549056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320927.452741:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320927.452745:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320927.452747:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320927.452751:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6170 00000100:00080000:1.0:1767320927.452754:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320927.452758:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320927.452776:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985549184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320927.452829:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320927.452834:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320927.452836:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320927.452841:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6170 00000100:00080000:1.0:1767320927.452844:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320927.452847:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320927.452861:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05500 x1853163985549440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320927.452879:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320927.452883:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320927.452885:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320927.452889:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6170 00000100:00080000:1.0:1767320927.452892:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320927.452896:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320927.452908:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05180 x1853163985549696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320927.453102:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985549056:0@lo:38:kworker.0 00000100:00100000:0.0:1767320927.453224:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320927.453307:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985549184:0@lo:38:kworker.0 00000100:00100000:0.0:1767320927.453334:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320927.453358:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985549440:0@lo:38:kworker.0 00000100:00100000:0.0:1767320927.453377:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320927.453391:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985549696:0@lo:38:kworker.0 00000100:00100000:0.0:1767320927.453408:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320927.453445:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549184 00000100:00100000:0.0:1767320927.453456:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549184 00000100:00100000:0.0:1767320927.453502:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2063 00000100:00100000:0.0:1767320927.453512:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985549184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320927.453537:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320927.453587:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985549184:12345-0@lo:38:kworker.0 Request processed in 74us (250us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320927.453596:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2063 00000100:00080000:0.0:1767320927.453665:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320927.453670:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320927.453673:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985549184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320927.453960:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985549952:0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454001:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320927.454040:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320927.454046:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985549824:0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454067:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320927.454094:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549952 00000100:00100000:3.0:1767320927.454098:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549952 00000100:00100000:3.0:1767320927.454146:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549824 00000100:00100000:3.0:1767320927.454150:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549824 00000100:00100000:3.0:1767320927.454200:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3569 00000100:00100000:3.0:1767320927.454208:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985549952:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454270:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985549952:12345-0@lo:400:kworker.0 Request processed in 63us (268us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320927.454277:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3569 00000100:00100000:3.0:1767320927.454291:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3570 00000100:00100000:3.0:1767320927.454298:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985549824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454325:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985549824:12345-0@lo:400:kworker.0 Request processed in 28us (257us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320927.454330:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3570 00000100:00100000:3.0:1767320927.454366:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985549824:0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454376:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985549952:0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454401:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985549568:0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454427:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320927.454452:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320927.454458:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985549312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454482:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320927.454510:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549056 00000100:00100000:3.0:1767320927.454514:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549056 00000100:00100000:3.0:1767320927.454534:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549440 00000100:00100000:3.0:1767320927.454537:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549440 00000100:00100000:3.0:1767320927.454544:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549696 00000100:00100000:3.0:1767320927.454546:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549696 00000100:00100000:3.0:1767320927.454553:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549568 00000100:00100000:3.0:1767320927.454555:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549568 00000100:00100000:3.0:1767320927.454571:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985549312 00000100:00100000:3.0:1767320927.454573:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985549312 00000100:00100000:3.0:1767320927.454585:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5513 00000100:00100000:3.0:1767320927.454591:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985549568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454624:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985549568:12345-0@lo:400:kworker.0 Request processed in 33us (196us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320927.454631:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5513 00000100:00100000:3.0:1767320927.454642:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5514 00000100:00100000:3.0:1767320927.454648:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985549312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454674:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985549312:12345-0@lo:400:kworker.0 Request processed in 26us (192us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320927.454680:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5514 00000100:00100000:3.0:1767320927.454691:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6289 00000100:00100000:3.0:1767320927.454695:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985549056:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320927.454709:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320927.454736:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985549056:12345-0@lo:38:kworker.0 Request processed in 39us (1513us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320927.454741:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6289 00000100:00100000:3.0:1767320927.454751:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6290 00000100:00100000:3.0:1767320927.454754:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985549440:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320927.454762:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320927.454783:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985549440:12345-0@lo:38:kworker.0 Request processed in 28us (1406us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320927.454788:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6290 00000100:00100000:3.0:1767320927.454798:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6291 00000100:00100000:3.0:1767320927.454801:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985549696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320927.454810:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320927.454832:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985549696:12345-0@lo:38:kworker.0 Request processed in 29us (1423us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320927.454838:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6291 00000100:00100000:3.0:1767320927.454868:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985549312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320927.454877:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985549568:0@lo:400:kworker.0 00000100:00080000:0.0:1767320927.459864:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320927.459884:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320927.459891:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985549056:0@lo:38:kworker.0 00000100:00080000:0.0:1767320927.459909:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320927.459914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320927.459917:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985549440:0@lo:38:kworker.0 00000100:00080000:0.0:1767320927.459929:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320927.459932:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320927.459935:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985549696:0@lo:38:kworker.0 00000100:00080000:0.0:1767320927.459981:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320927.459989:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320927.459993:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320927.964213:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985550208:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320927.964324:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320927.964423:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320927.964431:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985550080:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320927.964466:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320927.974336:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550208 00000100:00100000:0.0:1767320927.974360:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550208 00000100:00100000:2.0:1767320927.974792:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550080 00000100:00100000:2.0:1767320927.974799:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550080 00000100:00100000:2.0:1767320927.974915:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2786 00000100:00100000:2.0:1767320927.974929:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985550080:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320927.974976:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320927.975108:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985550080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 182us (10640us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320927.975117:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2786 00000100:00100000:3.0:1767320927.975266:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985550080:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320927.976852:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2787 00000100:00100000:1.0:1767320927.976869:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985550208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320927.976908:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320927.977011:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985550208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 146us (12686us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320927.977019:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2787 00000100:00100000:3.0:1767320927.977154:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985550208:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320928.091623:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320928.426895:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320928.428469:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644480 00000100:00100000:1.0:1767320928.428489:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644480 00000100:00100000:1.0:1767320928.428578:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6292 00000100:00100000:1.0:1767320928.428588:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920644480:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320928.428620:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320928.428725:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920644480:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 136us (1839us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320928.428733:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6292 00000100:00100000:1.0:1767320928.431976:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320928.433947:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644608 00000100:00100000:2.0:1767320928.433968:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644608 00000100:00100000:2.0:1767320928.434141:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5515 00000100:00100000:2.0:1767320928.434154:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920644608:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320928.434304:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920644608:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 151us (2346us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320928.434313:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5515 00000100:00080000:0.0:1767320928.435881:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320929.120695:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320930.140287:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320931.164221:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320932.187484:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320932.572196:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320932.572210:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320932.572214:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320932.572218:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320932.572223:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320932.572236:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6175 00000100:00080000:0.0:1767320932.572242:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320932.572252:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320932.572313:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3800 x1853163985550336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320932.572331:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320932.572334:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320932.572335:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320932.572339:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6175 00000100:00080000:0.0:1767320932.572341:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320932.572344:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320932.572358:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1180 x1853163985550464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320932.572403:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320932.572407:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320932.572409:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320932.572413:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6175 00000100:00080000:0.0:1767320932.572415:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320932.572418:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320932.572431:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9f80 x1853163985550720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320932.572447:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320932.572450:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320932.572452:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320932.572456:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6175 00000100:00080000:0.0:1767320932.572458:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320932.572462:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320932.572474:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a552d180 x1853163985550976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320932.572523:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985550848:0@lo:400:kworker.0 00000100:00100000:0.0:1767320932.572555:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985551104:0@lo:400:kworker.0 00000100:00100000:1.0:1767320932.572621:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320932.572623:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320932.572636:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985550336:0@lo:38:kworker.0 00000100:00100000:1.0:1767320932.572701:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320932.572708:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985550592:0@lo:400:kworker.0 00000100:00100000:3.0:1767320932.572771:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985551232:0@lo:400:kworker.0 00000100:00100000:0.0:1767320932.573320:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985551104 00000100:00100000:0.0:1767320932.573332:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985551104 00000100:00100000:0.0:1767320932.573406:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3571 00000100:00100000:0.0:1767320932.573418:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a03050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985551104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320932.573493:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a03050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985551104:12345-0@lo:400:kworker.0 Request processed in 76us (870us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320932.573503:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3571 00000100:00100000:2.0:1767320932.574912:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320932.575052:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985550464:0@lo:38:kworker.0 00000100:00100000:0.0:1767320932.575099:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985551104:0@lo:400:kworker.0 00000100:00080000:0.0:1767320932.575140:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320932.575200:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550848 00000100:00100000:0.0:1767320932.575207:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550848 00000100:00100000:2.0:1767320932.575241:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320932.575258:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550336 00000100:00100000:0.0:1767320932.575262:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550336 00000100:00100000:2.0:1767320932.575270:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985550720:0@lo:38:kworker.0 00000100:00100000:0.0:1767320932.575279:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5516 00000100:00100000:0.0:1767320932.575289:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985550848:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320932.575338:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985550848:12345-0@lo:400:kworker.0 Request processed in 51us (2719us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320932.575345:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5516 00000100:00100000:0.0:1767320932.575361:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6293 00000100:00100000:0.0:1767320932.575365:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985550336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320932.575387:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320932.575420:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985550336:12345-0@lo:38:kworker.0 Request processed in 53us (514us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320932.575426:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6293 00000100:00080000:0.0:1767320932.575442:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320932.575477:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550464 00000100:00100000:0.0:1767320932.575481:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550464 00000100:00100000:0.0:1767320932.575503:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2064 00000100:00100000:0.0:1767320932.575507:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985550464:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320932.575530:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767320932.575580:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320932.575608:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985550464:12345-0@lo:38:kworker.0 Request processed in 99us (366us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320932.575615:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2064 00000100:00100000:0.0:1767320932.575645:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550720 00000100:00100000:0.0:1767320932.575649:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550720 00000100:00100000:0.0:1767320932.575669:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6294 00000100:00100000:0.0:1767320932.575673:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985550720:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320932.575683:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320932.575711:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985550720:12345-0@lo:38:kworker.0 Request processed in 37us (181us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320932.575718:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6294 00000100:00100000:2.0:1767320932.575719:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985550976:0@lo:38:kworker.0 00000100:00100000:2.0:1767320932.575745:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320932.575766:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550976 00000100:00100000:0.0:1767320932.575770:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550976 00000100:00080000:2.0:1767320932.575777:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320932.575784:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320932.575785:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6295 00000100:00100000:0.0:1767320932.575788:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985550976:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320932.575797:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320932.575827:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985550976:12345-0@lo:38:kworker.0 Request processed in 37us (81us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320932.575833:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6295 00000100:00100000:2.0:1767320932.575874:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985550336:0@lo:38:kworker.0 00000100:00080000:2.0:1767320932.575890:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320932.575893:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320932.575896:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985550464:0@lo:38:kworker.0 00000100:00080000:2.0:1767320932.575903:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320932.575906:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320932.575909:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985550720:0@lo:38:kworker.0 00000100:00080000:2.0:1767320932.575915:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320932.575918:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320932.575921:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985550976:0@lo:38:kworker.0 00000100:00100000:1.0:1767320932.579872:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320932.580345:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985550848:0@lo:400:kworker.0 00000100:00100000:3.0:1767320932.580476:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320932.580604:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985551232 00000100:00100000:1.0:1767320932.580612:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985551232 00000100:00100000:1.0:1767320932.580691:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3572 00000100:00100000:1.0:1767320932.580701:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985551232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320932.580777:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985551232:12345-0@lo:400:kworker.0 Request processed in 77us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320932.580786:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3572 00000100:00100000:3.0:1767320932.580865:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985551232:0@lo:400:kworker.0 00000100:00080000:2.0:1767320932.581243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320932.583274:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985550592 00000100:00100000:0.0:1767320932.583292:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985550592 00000100:00100000:0.0:1767320932.583471:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5517 00000100:00100000:0.0:1767320932.583483:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985550592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320932.584365:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985550592:12345-0@lo:400:kworker.0 Request processed in 880us (4501us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320932.584377:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5517 00000100:00100000:3.0:1767320932.584525:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985550592:0@lo:400:kworker.0 00000100:00080000:2.0:1767320932.587672:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320933.084398:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320933.084435:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985551360:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320933.084530:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.084936:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985551488:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320933.084968:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320933.086760:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985551360 00000100:00100000:1.0:1767320933.086785:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985551360 00000100:00100000:1.0:1767320933.086906:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985551488 00000100:00100000:1.0:1767320933.086910:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985551488 00000100:00100000:1.0:1767320933.086954:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2788 00000100:00100000:1.0:1767320933.086966:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985551360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320933.087011:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320933.087103:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985551360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 140us (2573us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320933.087112:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2788 00000100:00100000:1.0:1767320933.087142:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2789 00000100:00100000:1.0:1767320933.087149:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985551488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320933.087187:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320933.087230:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985551488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (2249us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320933.087236:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2789 00000100:00100000:3.0:1767320933.087374:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985551488:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320933.087390:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985551360:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320933.211463:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320933.547941:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320933.548085:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320933.548216:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644864 00000100:00100000:3.0:1767320933.548232:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644864 00000100:00100000:3.0:1767320933.548316:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644736 00000100:00100000:3.0:1767320933.548320:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644736 00000100:00100000:3.0:1767320933.548341:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5518 00000100:00100000:3.0:1767320933.548350:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920644864:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320933.548432:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920644864:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 82us (518us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320933.548440:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5518 00000100:00100000:3.0:1767320933.548464:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6296 00000100:00100000:3.0:1767320933.548473:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920644736:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320933.548495:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320933.548530:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920644736:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (443us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320933.548537:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6296 00000100:00080000:0.0:1767320933.548682:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767320934.237057:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320935.261306:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320936.283462:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320937.321617:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320937.691821:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320937.691835:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320937.691839:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320937.691844:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320937.691850:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320937.691861:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6180 00000100:00080000:3.0:1767320937.691867:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320937.691876:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320937.691934:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d500 x1853163985551616/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320937.696318:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985551616:0@lo:38:kworker.0 00000100:00100000:1.0:1767320937.696428:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320937.696714:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985551616 00000100:00100000:0.0:1767320937.696727:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985551616 00000100:00100000:0.0:1767320937.696780:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6297 00000100:00100000:0.0:1767320937.696789:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985551616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320937.696818:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320937.696873:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985551616:12345-0@lo:38:kworker.0 Request processed in 84us (444us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320937.696880:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6297 00000100:00080000:2.0:1767320937.696933:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320937.696940:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320937.696945:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985551616:0@lo:38:kworker.0 00000100:00080000:3.0:1767320937.700061:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320937.700076:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320937.700081:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320937.700092:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6180 00000100:00080000:3.0:1767320937.700098:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320937.700107:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320937.700198:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d180 x1853163985551744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320937.700296:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320937.700300:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320937.700302:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320937.700307:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6180 00000100:00080000:3.0:1767320937.700309:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320937.700312:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320937.700325:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524d500 x1853163985552000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320937.700356:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320937.700359:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320937.700361:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320937.700365:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6180 00000100:00080000:3.0:1767320937.700368:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320937.700372:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320937.700382:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18d880 x1853163985552256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320937.700416:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985551872:0@lo:400:kworker.0 00000100:00100000:0.0:1767320937.700502:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320937.700517:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985552384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320937.700564:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320937.700576:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320937.700584:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985552128:0@lo:400:kworker.0 00000100:00100000:3.0:1767320937.700591:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320937.700600:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985552512:0@lo:400:kworker.0 00000100:00100000:0.0:1767320937.700607:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320937.700631:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320937.700645:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985551872 00000100:00100000:0.0:1767320937.700654:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985551872 00000100:00100000:3.0:1767320937.700699:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552128 00000100:00100000:3.0:1767320937.700704:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552128 00000100:00100000:0.0:1767320937.700732:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5519 00000100:00100000:0.0:1767320937.700741:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985551872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320937.700749:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5520 00000100:00100000:3.0:1767320937.700759:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985552128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320937.700807:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985551872:12345-0@lo:400:kworker.0 Request processed in 65us (305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320937.700809:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985552128:12345-0@lo:400:kworker.0 Request processed in 51us (200us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320937.700823:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5519 00000100:00100000:1.0:1767320937.700875:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552384 00000100:00100000:1.0:1767320937.700878:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552384 00000100:00100000:1.0:1767320937.700905:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552512 00000100:00100000:1.0:1767320937.700907:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552512 00000100:00100000:1.0:1767320937.700928:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3573 00000100:00100000:1.0:1767320937.700935:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985552384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320937.700978:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985552384:12345-0@lo:400:kworker.0 Request processed in 43us (413us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320937.700985:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3573 00000100:00100000:1.0:1767320937.701013:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3574 00000100:00100000:1.0:1767320937.701020:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985552512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320937.701057:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985552512:12345-0@lo:400:kworker.0 Request processed in 39us (428us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320937.701062:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3574 00000100:00100000:1.0:1767320937.701105:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985552512:0@lo:400:kworker.0 00000100:00100000:1.0:1767320937.701112:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985552384:0@lo:400:kworker.0 00000100:00080000:1.0:1767320937.701150:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320937.701156:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320937.701159:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320937.701220:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5520 00000100:00100000:1.0:1767320937.701770:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985552128:0@lo:400:kworker.0 00000100:00100000:1.0:1767320937.701785:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985551872:0@lo:400:kworker.0 00000100:00100000:2.0:1767320937.702504:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985551744:0@lo:38:kworker.0 00000100:00100000:2.0:1767320937.702577:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320937.702633:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985552000:0@lo:38:kworker.0 00000100:00100000:0.0:1767320937.702636:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985551744 00000100:00100000:0.0:1767320937.702645:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985551744 00000100:00100000:2.0:1767320937.702663:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320937.702683:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985552256:0@lo:38:kworker.0 00000100:00100000:0.0:1767320937.702693:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2065 00000100:00100000:0.0:1767320937.702703:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985551744:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320937.702710:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767320937.702731:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320937.702840:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985551744:12345-0@lo:38:kworker.0 Request processed in 136us (260us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320937.702848:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2065 00000100:00100000:3.0:1767320937.702871:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552000 00000100:00100000:3.0:1767320937.702877:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552000 00000100:00100000:1.0:1767320937.702922:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552256 00000100:00100000:1.0:1767320937.702927:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552256 00000100:00100000:3.0:1767320937.702927:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6298 00000100:00100000:3.0:1767320937.702935:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985552000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320937.702947:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320937.702959:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6299 00000100:00100000:1.0:1767320937.702966:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985552256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320937.702979:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320937.702984:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985552000:12345-0@lo:38:kworker.0 Request processed in 45us (319us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320937.702993:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6298 00000100:00100000:1.0:1767320937.703028:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985552256:12345-0@lo:38:kworker.0 Request processed in 61us (320us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320937.703046:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6299 00000100:00080000:2.0:1767320937.703426:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320937.703434:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320937.703438:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985551744:0@lo:38:kworker.0 00000100:00080000:2.0:1767320937.703448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320937.703452:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320937.703454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985552000:0@lo:38:kworker.0 00000100:00080000:2.0:1767320937.703462:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320937.703465:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320937.703469:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985552256:0@lo:38:kworker.0 00000100:00100000:3.0:1767320938.203795:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985552768:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320938.203905:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320938.203965:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320938.203971:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985552640:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320938.203993:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320938.204511:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552768 00000100:00100000:0.0:1767320938.204523:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552768 00000100:00100000:0.0:1767320938.204589:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552640 00000100:00100000:0.0:1767320938.204593:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552640 00000100:00100000:0.0:1767320938.204636:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2790 00000100:00100000:0.0:1767320938.204647:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985552768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320938.204688:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320938.204759:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985552768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 114us (854us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320938.204767:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2790 00000100:00100000:0.0:1767320938.204789:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2791 00000100:00100000:0.0:1767320938.204795:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985552640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320938.204812:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320938.204841:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985552640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (847us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320938.204848:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2791 00000100:00100000:0.0:1767320938.204901:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985552640:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320938.204919:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985552768:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320938.335611:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320938.667279:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320938.673132:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320938.687232:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920645120 00000100:00100000:0.0:1767320938.687256:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920645120 00000100:00100000:0.0:1767320938.687415:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920644992 00000100:00100000:0.0:1767320938.687420:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920644992 00000100:00100000:0.0:1767320938.687458:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5521 00000100:00100000:0.0:1767320938.687470:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920645120:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320938.687579:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920645120:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 112us (20317us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320938.687588:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5521 00000100:00100000:0.0:1767320938.687610:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6300 00000100:00100000:0.0:1767320938.687616:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920644992:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320938.687640:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320938.687669:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920644992:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (14554us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320938.687676:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6300 00000100:00080000:1.0:1767320938.688010:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320939.356073:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320940.380256:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320941.403450:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320942.427380:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320942.762716:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320942.762809:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920645504 00000100:00100000:2.0:1767320942.762820:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920645504 00000100:00100000:2.0:1767320942.762904:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5522 00000100:00100000:2.0:1767320942.762916:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920645504:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320942.763021:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920645504:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 107us (316us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320942.763028:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5522 00000400:00000100:0.0:1767320942.763292:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920645248 offset 0 length 520: 4 00000100:00100000:0.0:1767320942.763318:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320942.763783:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920645376 00000100:00100000:3.0:1767320942.763789:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920645376 00000100:00100000:3.0:1767320942.763830:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6301 00000100:00100000:3.0:1767320942.763836:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920645376:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320942.763858:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320942.763913:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920645376:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 75us (594us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320942.763921:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6301 00000100:00080000:1.0:1767320942.764319:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320942.811378:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320917/real 1767320917] req@ffff8ce1a55a1500 x1853163985546368/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320942 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320942.811413:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a55a1500 x1853163985546368/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320942 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320942.811417:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.811424:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320942.811429:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.811436:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.811441:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320942.811448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320942.811458:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6185 00000100:00080000:0.0:1767320942.811463:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320942.811469:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00080000:0.0:1767320942.811471:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320942.811475:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985546368:0@lo:250:kworker.0 00000100:00100000:0.0:1767320942.811514:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9180 x1853163985552896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.811536:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.811540:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.811541:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.811544:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6185 00000100:00080000:0.0:1767320942.811547:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.811550:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320942.811564:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7800 x1853163985553024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.811597:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.811600:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.811602:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.811605:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6185 00000100:00080000:0.0:1767320942.811607:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.811610:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320942.811622:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89880 x1853163985553280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.811643:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.811647:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.811649:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.811653:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6185 00000100:00080000:0.0:1767320942.811656:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.811660:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320942.811665:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985552896:0@lo:38:kworker.0 00000100:00100000:0.0:1767320942.811674:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b800 x1853163985553536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.811715:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.811718:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320942.811720:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.811723:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6165 00000100:00080000:0.0:1767320942.811726:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320942.811732:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320942.811734:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320942.811746:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a300 x1853163985553920/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320942.811770:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985553792:0@lo:400:kworker.0 00000100:00100000:2.0:1767320942.811786:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985553024:0@lo:38:kworker.0 00000100:00100000:0.0:1767320942.811809:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.811820:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320942.811836:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320942.811846:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985553664:0@lo:400:kworker.0 00000100:00100000:2.0:1767320942.811851:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985553280:0@lo:38:kworker.0 00000100:00100000:0.0:1767320942.811877:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.811885:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.811906:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985553536:0@lo:38:kworker.0 00000100:00100000:2.0:1767320942.811925:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.811953:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985553920:0@lo:250:kworker.0 00000400:00000100:2.0:1767320942.811973:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985553920 offset 0 length 520: 4 00000100:00100000:2.0:1767320942.812010:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985552896 00000100:00100000:2.0:1767320942.812018:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985552896 00000100:00100000:2.0:1767320942.812050:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985553280 00000100:00100000:2.0:1767320942.812054:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985553280 00000100:00100000:2.0:1767320942.812065:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985553536 00000100:00100000:2.0:1767320942.812068:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985553536 00000100:00100000:2.0:1767320942.812088:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6302 00000100:00100000:2.0:1767320942.812094:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985552896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320942.812114:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320942.812151:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6303 00000100:00100000:0.0:1767320942.812157:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985553280:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320942.812190:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985552896:12345-0@lo:38:kworker.0 Request processed in 93us (460us total) trans 0 rc -19/-19 00010000:00020000:0.0:1767320942.812193:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320942.812201:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6302 00000100:00100000:0.0:1767320942.812230:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985553280:12345-0@lo:38:kworker.0 Request processed in 70us (344us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320942.812234:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6304 00000100:00100000:0.0:1767320942.812245:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6303 00000100:00100000:2.0:1767320942.812246:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985553536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320942.812260:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320942.812291:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985553536:12345-0@lo:38:kworker.0 Request processed in 47us (364us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320942.812298:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6304 00000100:00080000:2.0:1767320942.812331:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.812336:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320942.812340:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985553536:0@lo:38:kworker.0 00000100:00080000:2.0:1767320942.812352:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.812356:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320942.812813:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985553152:0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.812847:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320942.812877:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320942.812883:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985553408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.812902:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320942.812928:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985553152 00000100:00100000:3.0:1767320942.812932:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985553152 00000100:00100000:3.0:1767320942.812975:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985553408 00000100:00100000:3.0:1767320942.812978:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985553408 00000100:00100000:3.0:1767320942.812989:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5523 00000100:00100000:3.0:1767320942.812994:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985553152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.813045:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985553152:12345-0@lo:400:kworker.0 Request processed in 50us (197us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320942.813051:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5523 00000100:00100000:3.0:1767320942.813064:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5524 00000100:00100000:3.0:1767320942.813069:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985553408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.813089:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985553408:12345-0@lo:400:kworker.0 Request processed in 21us (187us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320942.813095:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5524 00000100:00100000:3.0:1767320942.813126:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985553408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.813136:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985553152:0@lo:400:kworker.0 00000100:00100000:2.0:1767320942.814379:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985552896:0@lo:38:kworker.0 00000100:00080000:2.0:1767320942.814488:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.814494:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320942.814499:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985553280:0@lo:38:kworker.0 00000100:00100000:3.0:1767320942.815998:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985553792 00000100:00100000:3.0:1767320942.816011:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985553792 00000100:00100000:3.0:1767320942.816069:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985553664 00000100:00100000:3.0:1767320942.816072:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985553664 00000100:00100000:1.0:1767320942.816159:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985553024 00000100:00100000:1.0:1767320942.816182:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985553024 00000100:00100000:1.0:1767320942.816312:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2066 00000100:00100000:1.0:1767320942.816322:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985553024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320942.816347:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320942.816350:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320942.816356:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320942.816358:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320942.816422:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3575 00000100:00100000:3.0:1767320942.816431:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985553792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.816536:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985553792:12345-0@lo:400:kworker.0 Request processed in 107us (4728us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320942.816543:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3575 00000100:00100000:3.0:1767320942.816559:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3576 00000100:00100000:3.0:1767320942.816565:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985553664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.816586:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985553664:12345-0@lo:400:kworker.0 Request processed in 22us (4711us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320942.816592:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3576 00000100:00100000:2.0:1767320942.816899:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985553664:0@lo:400:kworker.0 00000100:00100000:2.0:1767320942.816912:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985553792:0@lo:400:kworker.0 00000100:00080000:2.0:1767320942.816983:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.816988:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320942.816991:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985553024:0@lo:38:kworker.0 00000100:00100000:1.0:1767320942.817018:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985553024:12345-0@lo:38:kworker.0 Request processed in 694us (5195us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320942.817026:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2066 00000100:00100000:2.0:1767320943.323477:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320943.323507:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985554048:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320943.323582:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320943.323701:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554048 00000100:00100000:0.0:1767320943.323710:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554048 00000100:00100000:0.0:1767320943.323768:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2792 00000100:00100000:0.0:1767320943.323778:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985554048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320943.323807:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320943.323857:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985554048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320943.323865:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2792 00000100:00100000:0.0:1767320943.323912:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985554048:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320943.324672:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320943.324689:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985554176:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320943.324735:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320943.324917:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554176 00000100:00100000:2.0:1767320943.324924:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554176 00000100:00100000:2.0:1767320943.324995:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2793 00000100:00100000:2.0:1767320943.325004:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985554176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320943.325026:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320943.325077:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985554176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (341us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320943.325085:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2793 00000100:00100000:1.0:1767320943.325379:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985554176:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320943.452549:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320944.475416:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320945.501192:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320946.524972:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320947.547458:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320947.884558:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320947.884630:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320947.884821:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920645632 00000100:00100000:3.0:1767320947.884832:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920645632 00000100:00100000:2.0:1767320947.884866:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920645760 00000100:00100000:2.0:1767320947.884871:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920645760 00000100:00100000:3.0:1767320947.884890:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6305 00000100:00100000:3.0:1767320947.884898:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920645632:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320947.884923:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320947.884929:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5525 00000100:00100000:2.0:1767320947.884937:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920645760:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320947.884992:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920645632:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 93us (445us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320947.885000:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6305 00000100:00080000:1.0:1767320947.885141:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320947.888949:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920645760:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 4000us (4304us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320947.888964:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5525 00000100:00080000:0.0:1767320947.931238:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320947.931250:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320947.931253:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320947.931257:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320947.931262:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320947.931270:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6191 00000100:00080000:0.0:1767320947.931275:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320947.931282:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320947.931323:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579500 x1853163985554304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320947.931351:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320947.931354:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320947.931357:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320947.931360:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6191 00000100:00080000:0.0:1767320947.931362:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320947.931366:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320947.931379:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b480 x1853163985554432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320947.931528:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985554304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.931593:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320947.931669:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985554432:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.931696:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320947.931761:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554432 00000100:00100000:2.0:1767320947.931769:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554432 00000100:00100000:2.0:1767320947.931801:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2067 00000100:00100000:2.0:1767320947.931809:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985554432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320947.931829:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320947.931883:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985554432:12345-0@lo:38:kworker.0 Request processed in 73us (185us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320947.931889:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2067 00000100:00080000:2.0:1767320947.931929:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.931934:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.931937:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985554432:0@lo:38:kworker.0 00000100:00100000:3.0:1767320947.932298:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554304 00000100:00100000:3.0:1767320947.932303:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554304 00000100:00100000:3.0:1767320947.932335:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6306 00000100:00100000:3.0:1767320947.932340:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985554304:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320947.932354:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320947.932388:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985554304:12345-0@lo:38:kworker.0 Request processed in 46us (794us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320947.932395:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6306 00000100:00080000:0.0:1767320947.933284:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320947.933291:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320947.933295:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320947.933300:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6191 00000100:00080000:0.0:1767320947.933303:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320947.933309:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320947.933336:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357b480 x1853163985554688/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320947.933418:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985554688:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.933462:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320947.933496:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.933500:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.933504:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985554304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.933539:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554688 00000100:00100000:2.0:1767320947.933544:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554688 00000100:00100000:2.0:1767320947.933577:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6307 00000100:00100000:2.0:1767320947.933583:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985554688:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320947.933613:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320947.933657:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985554688:12345-0@lo:38:kworker.0 Request processed in 72us (194us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320947.933663:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6307 00000100:00080000:2.0:1767320947.933692:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.933696:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.933699:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985554688:0@lo:38:kworker.0 00000100:00100000:3.0:1767320947.933879:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985554816:0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.933910:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320947.933939:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320947.933947:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985554560:0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.933971:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320947.933996:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554816 00000100:00100000:3.0:1767320947.934001:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554816 00000100:00100000:3.0:1767320947.934049:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554560 00000100:00100000:3.0:1767320947.934052:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554560 00000100:00100000:3.0:1767320947.934076:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5526 00000100:00100000:3.0:1767320947.934083:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985554816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.934128:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985554816:12345-0@lo:400:kworker.0 Request processed in 45us (217us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320947.934148:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5526 00000100:00100000:3.0:1767320947.934184:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5527 00000100:00100000:3.0:1767320947.934192:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985554560:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.934220:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985554560:12345-0@lo:400:kworker.0 Request processed in 30us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320947.934226:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5527 00000100:00100000:3.0:1767320947.934264:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985554560:0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.934274:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985554816:0@lo:400:kworker.0 00000100:00080000:3.0:1767320947.934308:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320947.941837:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320947.941856:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320947.941860:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320947.941869:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6191 00000100:00080000:0.0:1767320947.941874:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320947.941881:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320947.941923:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe23480 x1853163985554944/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320947.942009:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985554944:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.942071:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320947.942255:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985554944 00000100:00100000:3.0:1767320947.942266:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985554944 00000100:00100000:3.0:1767320947.942322:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6308 00000100:00100000:3.0:1767320947.942330:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985554944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320947.942350:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320947.944996:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985555200:0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.945057:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320947.945098:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320947.945104:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985555072:0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.945127:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320947.945192:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555200 00000100:00100000:0.0:1767320947.945201:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555200 00000100:00100000:0.0:1767320947.945256:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555072 00000100:00100000:0.0:1767320947.945260:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555072 00000100:00100000:0.0:1767320947.945280:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3577 00000100:00100000:0.0:1767320947.945288:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985555200:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.945336:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985555200:12345-0@lo:400:kworker.0 Request processed in 49us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320947.945344:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3577 00000100:00100000:0.0:1767320947.945362:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3578 00000100:00100000:0.0:1767320947.945368:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985555072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.945392:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985555072:12345-0@lo:400:kworker.0 Request processed in 25us (264us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320947.945398:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3578 00000100:00080000:0.0:1767320947.945426:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320947.945430:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320947.945460:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985555072:0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.945472:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985555200:0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.955802:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985554944:12345-0@lo:38:kworker.0 Request processed in 13460us (13719us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320947.955818:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6308 00000100:00080000:2.0:1767320947.956280:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.956286:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.956291:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985554944:0@lo:38:kworker.0 00000100:00100000:2.0:1767320948.443733:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985555328:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320948.443805:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320948.443894:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320948.443900:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985555456:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320948.443925:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320948.444006:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555328 00000100:00100000:1.0:1767320948.444014:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555328 00000100:00100000:1.0:1767320948.444063:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555456 00000100:00100000:1.0:1767320948.444066:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555456 00000100:00100000:1.0:1767320948.444087:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2794 00000100:00100000:1.0:1767320948.444096:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985555328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320948.444134:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320948.444203:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985555328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320948.444214:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2794 00000100:00100000:1.0:1767320948.444229:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2795 00000100:00100000:1.0:1767320948.444236:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985555456:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320948.444251:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320948.444275:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985555456:12345-0@lo:13:osp-pre-0-1.0 Request processed in 40us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320948.444281:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2795 00000100:00100000:2.0:1767320948.445680:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985555456:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320948.445699:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985555328:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320948.572345:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320949.595924:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320950.619656:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320951.643464:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320952.670499:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320953.012540:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320953.012617:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320953.012623:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920645888 00000100:00100000:2.0:1767320953.012648:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920645888 00000100:00100000:2.0:1767320953.012709:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920646016 00000100:00100000:2.0:1767320953.012712:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920646016 00000100:00100000:2.0:1767320953.012772:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5528 00000100:00100000:2.0:1767320953.012787:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920646016:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767320953.012879:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920646016:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 97us (263us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320953.012886:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5528 00000100:00100000:2.0:1767320953.012907:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6309 00000100:00100000:2.0:1767320953.012911:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920645888:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320953.012931:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320953.012971:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920645888:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 58us (448us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320953.012978:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6309 00000100:00080000:2.0:1767320953.012995:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320953.051317:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320953.051325:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320953.051329:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320953.051332:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320953.051335:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320953.051343:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6196 00000100:00080000:2.0:1767320953.051347:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320953.051352:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320953.051390:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff800 x1853163985555584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320953.051412:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320953.051414:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320953.051415:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320953.051418:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6196 00000100:00080000:2.0:1767320953.051419:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320953.051421:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320953.051432:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff100 x1853163985555712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320953.051486:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985555840:0@lo:400:kworker.0 00000100:00100000:1.0:1767320953.051541:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320953.051645:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985555584:0@lo:38:kworker.0 00000100:00100000:3.0:1767320953.051659:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555840 00000100:00100000:0.0:1767320953.051669:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320953.051678:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555840 00000100:00100000:0.0:1767320953.051709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985555712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320953.051728:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320953.051791:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555584 00000100:00100000:3.0:1767320953.051798:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555584 00000100:00100000:0.0:1767320953.051805:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555712 00000100:00100000:0.0:1767320953.051814:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555712 00000100:00100000:3.0:1767320953.051830:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5529 00000100:00100000:3.0:1767320953.051845:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985555840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320953.051854:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2068 00000100:00100000:0.0:1767320953.051861:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985555712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320953.051879:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320953.051954:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320953.051957:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320953.051959:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320953.051961:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6196 00000100:00080000:2.0:1767320953.051963:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320953.051965:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320953.051976:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffd880 x1853163985555968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320953.051999:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985555712:12345-0@lo:38:kworker.0 Request processed in 70us (202us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320953.052004:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2068 00000100:00100000:0.0:1767320953.052026:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985556096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320953.052043:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320953.052063:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985555968:0@lo:38:kworker.0 00000100:00100000:0.0:1767320953.052076:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320953.052097:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320953.052101:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320953.052105:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985555712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320953.052141:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556096 00000100:00100000:0.0:1767320953.052145:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556096 00000100:00100000:0.0:1767320953.052193:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985555968 00000100:00100000:0.0:1767320953.052195:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985555968 00000100:00100000:0.0:1767320953.052203:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5530 00000100:00100000:0.0:1767320953.052208:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985556096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320953.052237:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985556096:12345-0@lo:400:kworker.0 Request processed in 30us (194us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320953.052242:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5530 00000100:00100000:0.0:1767320953.052250:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6310 00000100:00100000:0.0:1767320953.052252:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985555584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320953.052260:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320953.052275:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985555584:12345-0@lo:38:kworker.0 Request processed in 22us (608us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320953.052279:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6310 00000100:00100000:0.0:1767320953.052287:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6311 00000100:00100000:0.0:1767320953.052289:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985555968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320953.052294:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320953.052304:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985555968:12345-0@lo:38:kworker.0 Request processed in 15us (228us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320953.052308:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6311 00000100:00100000:0.0:1767320953.052323:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985556096:0@lo:400:kworker.0 00000100:00080000:0.0:1767320953.052334:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320953.052337:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320953.052338:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985555968:0@lo:38:kworker.0 00000100:00080000:0.0:1767320953.052343:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320953.052344:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320953.052346:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985555584:0@lo:38:kworker.0 00000100:00080000:0.0:1767320953.052367:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320953.055359:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320953.055368:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320953.055372:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320953.055380:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6196 00000100:00080000:2.0:1767320953.055385:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320953.055391:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320953.055427:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985555840:12345-0@lo:400:kworker.0 Request processed in 3573us (3869us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320953.055431:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff800 x1853163985556224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320953.055474:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5529 00000100:00100000:2.0:1767320953.055536:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985556352:0@lo:400:kworker.0 00000100:00100000:3.0:1767320953.055555:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985556480:0@lo:400:kworker.0 00000100:00100000:2.0:1767320953.055596:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320953.055633:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320953.055811:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985556224:0@lo:38:kworker.0 00000100:00100000:0.0:1767320953.055843:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320953.056127:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985555840:0@lo:400:kworker.0 00000100:00100000:1.0:1767320953.056192:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556224 00000100:00100000:1.0:1767320953.056197:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556224 00000100:00100000:1.0:1767320953.056223:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6312 00000100:00100000:1.0:1767320953.056228:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985556224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320953.056246:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320953.056279:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985556224:12345-0@lo:38:kworker.0 Request processed in 51us (436us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320953.056284:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6312 00000100:00080000:1.0:1767320953.056303:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320953.056306:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320953.056308:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985556224:0@lo:38:kworker.0 00000100:00100000:2.0:1767320953.056390:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556352 00000100:00100000:2.0:1767320953.056393:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556352 00000100:00100000:2.0:1767320953.056460:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556480 00000100:00100000:2.0:1767320953.056462:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556480 00000100:00100000:1.0:1767320953.056490:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3579 00000100:00100000:1.0:1767320953.056496:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985556352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320953.056527:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985556352:12345-0@lo:400:kworker.0 Request processed in 32us (932us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320953.056531:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3579 00000100:00080000:0.0:1767320953.056566:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320953.056568:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320953.056628:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3580 00000100:00100000:2.0:1767320953.056634:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985556480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320953.056660:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985556480:12345-0@lo:400:kworker.0 Request processed in 27us (1033us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320953.056664:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3580 00000100:00100000:2.0:1767320953.056681:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985556352:0@lo:400:kworker.0 00000100:00100000:1.0:1767320953.056824:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985556480:0@lo:400:kworker.0 00000100:00100000:2.0:1767320953.564347:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320953.564372:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985556608:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320953.564440:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320953.566872:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985556736:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320953.566944:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320953.567052:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556608 00000100:00100000:1.0:1767320953.567061:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556608 00000100:00100000:1.0:1767320953.567113:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556736 00000100:00100000:1.0:1767320953.567116:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556736 00000100:00100000:1.0:1767320953.567136:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2796 00000100:00100000:1.0:1767320953.567144:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985556608:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320953.567200:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320953.567248:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985556608:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (2807us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320953.567255:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2796 00000100:00100000:1.0:1767320953.567272:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2797 00000100:00100000:1.0:1767320953.567279:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985556736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320953.567294:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320953.567320:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985556736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (384us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320953.567325:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2797 00000100:00100000:1.0:1767320953.567371:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985556736:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320953.567387:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985556608:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320953.693624:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320954.722733:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320955.741402:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320956.764442:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320957.787485:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320958.122794:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320958.122913:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320958.127449:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920646272 00000100:00100000:3.0:1767320958.127463:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920646272 00000100:00100000:3.0:1767320958.127604:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920646144 00000100:00100000:3.0:1767320958.127608:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920646144 00000100:00100000:3.0:1767320958.127632:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5531 00000100:00100000:3.0:1767320958.127643:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920646272:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320958.127750:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920646272:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 109us (4966us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320958.127758:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5531 00000100:00100000:3.0:1767320958.127776:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6313 00000100:00100000:3.0:1767320958.127781:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920646144:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320958.127801:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320958.127825:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920646144:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (4912us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320958.127831:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6313 00000100:00080000:0.0:1767320958.128636:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320958.173079:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320958.173089:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320958.173092:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320958.173095:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320958.173099:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320958.173107:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6201 00000100:00080000:3.0:1767320958.173112:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320958.173117:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320958.173159:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb100 x1853163985556864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320958.173212:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320958.173215:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320958.173216:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320958.173219:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6201 00000100:00080000:3.0:1767320958.173223:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320958.173226:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320958.173239:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdaa00 x1853163985556992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320958.173278:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320958.173282:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320958.173284:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320958.173287:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6201 00000100:00080000:3.0:1767320958.173290:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320958.173293:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320958.173306:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9880 x1853163985557248/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320958.173322:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320958.173325:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320958.173327:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320958.173330:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6201 00000100:00080000:3.0:1767320958.173332:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320958.173335:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320958.173345:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8700 x1853163985557504/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320958.173389:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985557632:0@lo:400:kworker.0 00000100:00100000:3.0:1767320958.173449:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320958.173509:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320958.173516:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985557760:0@lo:400:kworker.0 00000100:00100000:3.0:1767320958.173541:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320958.173569:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985557632 00000100:00100000:3.0:1767320958.173576:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985557632 00000100:00100000:3.0:1767320958.173622:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985557760 00000100:00100000:3.0:1767320958.173626:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985557760 00000100:00100000:3.0:1767320958.173649:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3581 00000100:00100000:3.0:1767320958.173667:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985557632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320958.173718:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985557632:12345-0@lo:400:kworker.0 Request processed in 61us (268us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320958.173725:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3581 00000100:00100000:3.0:1767320958.173742:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3582 00000100:00100000:3.0:1767320958.173748:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985557760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320958.173774:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985557760:12345-0@lo:400:kworker.0 Request processed in 27us (233us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320958.173779:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3582 00000100:00100000:3.0:1767320958.173815:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985557760:0@lo:400:kworker.0 00000100:00100000:3.0:1767320958.173826:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985557632:0@lo:400:kworker.0 00000100:00100000:0.0:1767320958.173906:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985557376:0@lo:400:kworker.0 00000100:00100000:0.0:1767320958.173937:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320958.173969:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320958.173974:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985557120:0@lo:400:kworker.0 00000100:00100000:0.0:1767320958.173994:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320958.174016:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320958.174020:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320958.174137:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985556864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320958.174192:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320958.174223:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985556992:0@lo:38:kworker.0 00000100:00100000:1.0:1767320958.174246:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320958.174266:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985557248:0@lo:38:kworker.0 00000100:00100000:1.0:1767320958.174285:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320958.174300:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985557504:0@lo:38:kworker.0 00000100:00100000:1.0:1767320958.174318:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320958.174394:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985557376 00000100:00100000:2.0:1767320958.174399:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985557376 00000100:00100000:2.0:1767320958.174440:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985557120 00000100:00100000:2.0:1767320958.174444:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985557120 00000100:00100000:2.0:1767320958.174456:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556864 00000100:00100000:2.0:1767320958.174459:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556864 00000100:00100000:2.0:1767320958.174466:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985557248 00000100:00100000:2.0:1767320958.174468:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985557248 00000100:00100000:2.0:1767320958.174473:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985557504 00000100:00100000:2.0:1767320958.174475:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985557504 00000100:00100000:2.0:1767320958.174484:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5532 00000100:00100000:2.0:1767320958.174490:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985557376:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320958.174530:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985557376:12345-0@lo:400:kworker.0 Request processed in 39us (591us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320958.174536:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5532 00000100:00100000:2.0:1767320958.174549:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5533 00000100:00100000:2.0:1767320958.174555:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985557120:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320958.174580:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985557120:12345-0@lo:400:kworker.0 Request processed in 25us (584us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320958.174585:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5533 00000100:00100000:2.0:1767320958.174596:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6314 00000100:00100000:2.0:1767320958.174600:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985556864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320958.174616:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320958.174644:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985556864:12345-0@lo:38:kworker.0 Request processed in 43us (452us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320958.174650:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6314 00000100:00100000:2.0:1767320958.174671:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6315 00000100:00100000:2.0:1767320958.174674:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985557248:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320958.174684:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320958.174705:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985557248:12345-0@lo:38:kworker.0 Request processed in 28us (418us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320958.174710:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6315 00000100:00100000:2.0:1767320958.174720:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6316 00000100:00100000:2.0:1767320958.174723:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985557504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320958.174730:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320958.174749:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985557504:12345-0@lo:38:kworker.0 Request processed in 25us (431us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320958.174754:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6316 00000100:00080000:2.0:1767320958.174781:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320958.174786:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320958.174789:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985556864:0@lo:38:kworker.0 00000100:00080000:2.0:1767320958.174797:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320958.174801:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320958.174803:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985557248:0@lo:38:kworker.0 00000100:00080000:2.0:1767320958.174809:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320958.174812:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320958.174814:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985557504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320958.174844:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985557120:0@lo:400:kworker.0 00000100:00100000:2.0:1767320958.174853:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985557376:0@lo:400:kworker.0 00000100:00100000:0.0:1767320958.174924:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985556992 00000100:00100000:0.0:1767320958.174928:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985556992 00000100:00100000:0.0:1767320958.174958:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2069 00000100:00100000:0.0:1767320958.174963:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985556992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320958.174973:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320958.175000:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985556992:12345-0@lo:38:kworker.0 Request processed in 35us (753us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320958.175006:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2069 00000100:00080000:2.0:1767320958.175081:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320958.175085:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320958.175087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985556992:0@lo:38:kworker.0 00000100:00080000:1.0:1767320958.175128:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320958.685009:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985557888:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320958.685076:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320958.685298:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985557888 00000100:00100000:1.0:1767320958.685307:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985557888 00000100:00100000:1.0:1767320958.685378:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2798 00000100:00100000:1.0:1767320958.685387:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985557888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320958.685417:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320958.685468:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985557888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (392us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320958.685476:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2798 00000100:00100000:1.0:1767320958.685522:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985557888:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320958.686423:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320958.687845:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985558016:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320958.687916:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320958.688140:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558016 00000100:00100000:2.0:1767320958.688149:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558016 00000100:00100000:2.0:1767320958.688253:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2799 00000100:00100000:2.0:1767320958.688263:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985558016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320958.688293:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320958.688367:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985558016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 106us (451us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320958.688375:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2799 00000100:00100000:3.0:1767320958.688665:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985558016:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767320958.811501:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320959.835412:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320960.864289:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320961.884195:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320962.907408:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320963.245209:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320963.245269:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320963.245504:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920646528 00000100:00100000:3.0:1767320963.245515:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920646528 00000100:00100000:3.0:1767320963.245620:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920646400 00000100:00100000:3.0:1767320963.245624:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920646400 00000100:00100000:3.0:1767320963.245643:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5534 00000100:00100000:3.0:1767320963.245655:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920646528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320963.245738:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920646528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (574us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320963.245746:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5534 00000100:00100000:3.0:1767320963.245765:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6317 00000100:00100000:3.0:1767320963.245769:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920646400:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320963.245790:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320963.245816:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920646400:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (546us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320963.245822:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6317 00000100:00080000:1.0:1767320963.246270:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320963.292186:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320963.292197:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320963.292200:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320963.292204:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320963.292209:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320963.292218:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6206 00000100:00080000:0.0:1767320963.292224:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320963.293302:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320963.293347:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9500 x1853163985558144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320963.293375:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320963.293379:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320963.293381:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320963.293385:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6206 00000100:00080000:0.0:1767320963.293388:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320963.293392:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320963.293408:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9880 x1853163985558272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320963.293453:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320963.293457:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320963.293459:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320963.293462:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6206 00000100:00080000:0.0:1767320963.293464:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320963.293467:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320963.293479:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985558528/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320963.293494:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320963.293496:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320963.293498:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320963.293501:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6206 00000100:00080000:0.0:1767320963.293503:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320963.293506:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320963.293514:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60700 x1853163985558784/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320963.293592:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985558912:0@lo:400:kworker.0 00000100:00100000:1.0:1767320963.293598:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985558656:0@lo:400:kworker.0 00000100:00100000:3.0:1767320963.293683:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985558400:0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.293782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985558144:0@lo:38:kworker.0 00000100:00100000:1.0:1767320963.295979:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320963.296076:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320963.296115:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985559040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320963.296133:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558656 00000100:00100000:3.0:1767320963.296142:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558656 00000100:00100000:1.0:1767320963.296160:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320963.296213:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320963.296223:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558400 00000100:00100000:3.0:1767320963.296228:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558400 00000100:00100000:1.0:1767320963.296229:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985559040 00000100:00100000:1.0:1767320963.296235:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985559040 00000100:00100000:3.0:1767320963.296265:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5535 00000100:00100000:1.0:1767320963.296276:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3583 00000100:00100000:3.0:1767320963.296277:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985558656:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320963.296290:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985559040:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320963.296342:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985559040:12345-0@lo:400:kworker.0 Request processed in 57us (181us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320963.296350:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985558656:12345-0@lo:400:kworker.0 Request processed in 74us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320963.296355:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3583 00000100:00100000:3.0:1767320963.296361:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5535 00000100:00100000:3.0:1767320963.296382:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5536 00000100:00100000:3.0:1767320963.296391:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985558400:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320963.296396:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320963.296400:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320963.296431:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985558400:12345-0@lo:400:kworker.0 Request processed in 41us (357us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320963.296437:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5536 00000100:00100000:3.0:1767320963.296474:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985558400:0@lo:400:kworker.0 00000100:00100000:0.0:1767320963.296612:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320963.296661:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985558656:0@lo:400:kworker.0 00000100:00100000:1.0:1767320963.296724:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985559040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320963.303051:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558912 00000100:00100000:3.0:1767320963.303071:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558912 00000100:00100000:3.0:1767320963.303223:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3584 00000100:00100000:3.0:1767320963.303233:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985558912:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320963.303293:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320963.303342:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985558912:12345-0@lo:400:kworker.0 Request processed in 110us (6728us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320963.303351:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3584 00000100:00100000:0.0:1767320963.303618:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985558912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.305009:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320963.305180:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985558272:0@lo:38:kworker.0 00000100:00100000:2.0:1767320963.305221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320963.305244:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985558528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320963.305268:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320963.305325:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558272 00000100:00100000:0.0:1767320963.305334:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558272 00000100:00100000:0.0:1767320963.305376:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2070 00000100:00100000:0.0:1767320963.305384:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985558272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320963.305407:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320963.305449:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985558272:12345-0@lo:38:kworker.0 Request processed in 64us (226us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320963.305457:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2070 00000100:00100000:2.0:1767320963.308794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985558784:0@lo:38:kworker.0 00000100:00100000:2.0:1767320963.308854:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320963.308876:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558144 00000100:00100000:3.0:1767320963.308884:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558144 00000100:00100000:3.0:1767320963.308927:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558528 00000100:00100000:3.0:1767320963.308930:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558528 00000100:00100000:3.0:1767320963.308947:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6318 00000100:00100000:3.0:1767320963.308954:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985558144:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320963.308960:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6319 00000100:00100000:1.0:1767320963.308966:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985558528:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320963.308974:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320963.308975:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320963.309013:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985558528:12345-0@lo:38:kworker.0 Request processed in 45us (3743us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320963.309013:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985558144:12345-0@lo:38:kworker.0 Request processed in 58us (4014us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320963.309020:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6319 00000100:00100000:3.0:1767320963.309080:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6318 00000100:00080000:2.0:1767320963.311588:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320963.311600:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320963.311606:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985558144:0@lo:38:kworker.0 00000100:00080000:2.0:1767320963.311620:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320963.311624:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320963.311627:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985558272:0@lo:38:kworker.0 00000100:00080000:2.0:1767320963.311637:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320963.311641:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320963.311644:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985558528:0@lo:38:kworker.0 00000100:00100000:0.0:1767320963.312195:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985558784 00000100:00100000:0.0:1767320963.312204:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985558784 00000100:00100000:0.0:1767320963.312318:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6320 00000100:00100000:0.0:1767320963.312326:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985558784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320963.312347:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320963.312420:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985558784:12345-0@lo:38:kworker.0 Request processed in 93us (3565us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320963.312427:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6320 00000100:00080000:2.0:1767320963.312498:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320963.312503:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320963.312507:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985558784:0@lo:38:kworker.0 00000100:00100000:3.0:1767320963.803496:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985559168:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320963.803586:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320963.803827:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985559296:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320963.803860:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320963.806511:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985559168 00000100:00100000:2.0:1767320963.806525:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985559168 00000100:00100000:2.0:1767320963.806625:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985559296 00000100:00100000:2.0:1767320963.806629:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985559296 00000100:00100000:2.0:1767320963.806662:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2800 00000100:00100000:2.0:1767320963.806672:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985559168:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320963.806705:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320963.806769:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985559168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (3182us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320963.806777:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2800 00000100:00100000:2.0:1767320963.806796:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2801 00000100:00100000:2.0:1767320963.806802:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985559296:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320963.806816:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320963.806849:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985559296:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (2988us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320963.806858:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2801 00000100:00100000:3.0:1767320963.806963:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985559168:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320963.807011:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985559296:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320963.931568:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320964.956393:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320965.979395:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320967.003476:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320967.338912:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320967.339196:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920646912 00000100:00100000:0.0:1767320967.339207:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920646912 00000100:00100000:0.0:1767320967.339279:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5537 00000100:00100000:0.0:1767320967.339291:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920646912:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320967.339374:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920646912:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (472us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320967.339382:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5537 00000100:00080000:1.0:1767320967.339575:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767320967.342112:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920646656 offset 0 length 520: 4 00000100:00100000:3.0:1767320967.342151:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320967.342576:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920646784 00000100:00100000:0.0:1767320967.342586:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920646784 00000100:00100000:0.0:1767320967.342637:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6321 00000100:00100000:0.0:1767320967.342644:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920646784:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320967.342667:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320967.342737:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920646784:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (585us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320967.342745:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6321 00000100:00100000:1.0:1767320967.387320:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320942/real 1767320942] req@ffff8ce1b357a300 x1853163985553920/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320967 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320967.387354:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b357a300 x1853163985553920/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320967 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320967.387375:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320967.387549:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320967.387555:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985553920:0@lo:250:kworker.0 00000100:00080000:2.0:1767320967.387816:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320967.387821:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320967.387825:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320967.387830:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6191 00000100:00080000:2.0:1767320967.387834:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320967.387840:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320967.387878:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593ad80 x1853163985559424/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320967.387911:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320967.387915:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320967.387917:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320967.387921:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6211 00000100:00080000:2.0:1767320967.387925:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320967.387929:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320967.387948:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9880 x1853163985559552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320967.387957:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320967.387960:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320967.387962:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320967.387965:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6211 00000100:00080000:2.0:1767320967.387968:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320967.387972:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320967.387983:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9500 x1853163985559680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320967.388015:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320967.388019:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320967.388021:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320967.388025:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6211 00000100:00080000:2.0:1767320967.388027:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320967.388030:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320967.388042:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3b80 x1853163985559936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320967.388057:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320967.388060:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320967.388062:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320967.388065:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6211 00000100:00080000:2.0:1767320967.388067:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320967.388070:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320967.388081:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3800 x1853163985560192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320967.388267:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985560064:0@lo:400:kworker.0 00000100:00100000:3.0:1767320967.388508:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320967.388582:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320967.388588:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985559808:0@lo:400:kworker.0 00000100:00100000:3.0:1767320967.388614:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320967.388644:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985560320:0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.388829:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985560448:0@lo:400:kworker.0 00000100:00100000:1.0:1767320967.388898:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985559424:0@lo:250:kworker.0 00000100:00100000:2.0:1767320967.388949:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560064 00000100:00100000:2.0:1767320967.388958:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560064 00000100:00100000:2.0:1767320967.389014:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985559808 00000100:00100000:2.0:1767320967.389017:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985559808 00000100:00100000:2.0:1767320967.389039:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5538 00000100:00100000:2.0:1767320967.389052:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985560064:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.389429:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320967.389466:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560448 00000100:00100000:0.0:1767320967.389471:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560448 00000100:00100000:0.0:1767320967.389512:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3585 00000100:00100000:0.0:1767320967.389519:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985560448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.389555:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985560448:12345-0@lo:400:kworker.0 Request processed in 36us (125us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320967.389562:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3585 00000100:00100000:0.0:1767320967.389605:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985560448:0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.389638:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5539 00000100:00100000:0.0:1767320967.389647:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985559808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.389698:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985559808:12345-0@lo:400:kworker.0 Request processed in 52us (1082us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320967.389704:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5539 00000100:00100000:0.0:1767320967.389734:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985559808:0@lo:400:kworker.0 00000100:00080000:0.0:1767320967.389766:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320967.389770:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320967.390062:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000100:1.0:1767320967.390145:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985559424 offset 0 length 520: 4 00000100:00100000:1.0:1767320967.390196:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985559552:0@lo:38:kworker.0 00000100:00100000:1.0:1767320967.390224:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320967.390240:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985559680:0@lo:38:kworker.0 00000100:00100000:1.0:1767320967.390264:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320967.390281:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985559936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320967.390300:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320967.390315:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985560192:0@lo:38:kworker.0 00000100:00100000:1.0:1767320967.390334:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320967.390363:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985559680 00000100:00100000:1.0:1767320967.390367:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985559680 00000100:00100000:1.0:1767320967.390402:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2071 00000100:00100000:1.0:1767320967.390408:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985559680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320967.390425:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320967.390457:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985559680:12345-0@lo:38:kworker.0 Request processed in 47us (191us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320967.390463:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2071 00000100:00100000:1.0:1767320967.390483:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985559552 00000100:00100000:1.0:1767320967.390497:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985559552 00000100:00100000:1.0:1767320967.390515:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985559936 00000100:00100000:1.0:1767320967.390517:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985559936 00000100:00100000:1.0:1767320967.390528:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560192 00000100:00100000:1.0:1767320967.390531:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560192 00000100:00100000:1.0:1767320967.390540:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6322 00000100:00100000:1.0:1767320967.390545:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985559552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320967.390554:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320967.390578:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985559552:12345-0@lo:38:kworker.0 Request processed in 32us (353us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320967.390584:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6322 00000100:00100000:1.0:1767320967.390594:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6323 00000100:00100000:1.0:1767320967.390598:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985559936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320967.390606:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320967.390623:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985559936:12345-0@lo:38:kworker.0 Request processed in 25us (323us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320967.390629:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6323 00000100:00100000:1.0:1767320967.390638:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6324 00000100:00100000:1.0:1767320967.390642:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985560192:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320967.390650:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320967.390680:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985560192:12345-0@lo:38:kworker.0 Request processed in 36us (345us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320967.390685:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6324 00000100:00080000:1.0:1767320967.390717:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320967.390722:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320967.390726:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985559552:0@lo:38:kworker.0 00000100:00080000:1.0:1767320967.390734:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320967.390737:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320967.390740:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985559680:0@lo:38:kworker.0 00000100:00080000:1.0:1767320967.390746:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320967.390750:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320967.390752:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985559936:0@lo:38:kworker.0 00000100:00080000:1.0:1767320967.390758:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320967.390761:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320967.390764:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985560192:0@lo:38:kworker.0 00000100:00100000:0.0:1767320967.391035:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560320 00000100:00100000:0.0:1767320967.391040:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560320 00000100:00100000:0.0:1767320967.391088:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3586 00000100:00100000:0.0:1767320967.391095:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985560320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.391138:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985560320:12345-0@lo:400:kworker.0 Request processed in 43us (1076us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320967.391145:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3586 00000100:00100000:3.0:1767320967.391230:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985560320:0@lo:400:kworker.0 00000100:00080000:1.0:1767320967.391285:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320967.391405:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985560064:12345-0@lo:400:kworker.0 Request processed in 2357us (2896us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320967.391413:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5538 00000100:00100000:0.0:1767320967.391527:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985560064:0@lo:400:kworker.0 00010000:00010000:2.0:1767320968.029421:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320968.923780:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985560576:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320968.923848:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320968.923900:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320968.923907:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985560704:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320968.923928:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320968.924041:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560576 00000100:00100000:2.0:1767320968.924049:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560576 00000100:00100000:2.0:1767320968.924098:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560704 00000100:00100000:2.0:1767320968.924101:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560704 00000100:00100000:2.0:1767320968.924129:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2802 00000100:00100000:2.0:1767320968.924138:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985560576:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320968.924186:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2803 00002000:00000024:2.0:1767320968.924188:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320968.924194:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985560704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320968.924210:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320968.924238:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985560704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (308us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320968.924238:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985560576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (389us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320968.924245:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2803 00000100:00100000:1.0:1767320968.924317:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985560704:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320968.924329:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985560576:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320968.931805:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2802 00010000:00010000:2.0:1767320969.051654:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320970.076523:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320971.099555:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320972.124094:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320972.457492:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320972.457583:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647168 00000100:00100000:3.0:1767320972.457594:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647168 00000100:00100000:3.0:1767320972.457669:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5540 00000100:00100000:3.0:1767320972.457680:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920647168:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767320972.457950:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920647168:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 271us (468us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320972.457958:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5540 00000100:00080000:1.0:1767320972.458809:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320972.476310:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320972.476492:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647040 00000100:00100000:3.0:1767320972.476503:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647040 00000100:00100000:3.0:1767320972.476553:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6325 00000100:00100000:3.0:1767320972.476562:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920647040:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767320972.476586:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320972.476707:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920647040:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 144us (405us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320972.476715:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6325 00000100:00080000:3.0:1767320972.507308:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320972.507319:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320972.507323:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320972.507327:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320972.507331:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320972.507339:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6215 00000100:00080000:3.0:1767320972.507344:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320972.507351:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320972.507402:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985560832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320972.507463:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985560832:0@lo:38:kworker.0 00000100:00100000:1.0:1767320972.507530:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320972.507651:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560832 00000100:00100000:0.0:1767320972.507660:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560832 00000100:00100000:0.0:1767320972.507710:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6326 00000100:00100000:0.0:1767320972.507718:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985560832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320972.507738:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320972.509856:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320972.509866:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320972.509869:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320972.509876:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6215 00000100:00080000:3.0:1767320972.509881:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320972.509887:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320972.509921:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985560960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320972.519056:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985560960:0@lo:38:kworker.0 00000100:00100000:1.0:1767320972.519133:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320972.519234:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985560960 00000100:00100000:2.0:1767320972.519242:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985560960 00000100:00100000:2.0:1767320972.519274:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2072 00000100:00100000:2.0:1767320972.519280:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985560960:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320972.519299:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320972.519334:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985560960:12345-0@lo:38:kworker.0 Request processed in 54us (202us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320972.519340:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2072 00000100:00100000:0.0:1767320972.522235:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985560832:12345-0@lo:38:kworker.0 Request processed in 14505us (14693us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320972.522251:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6326 00000100:00080000:1.0:1767320972.523489:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320972.523500:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320972.523506:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985560832:0@lo:38:kworker.0 00000100:00080000:1.0:1767320972.523543:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320972.523547:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320972.523550:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985560960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320972.524412:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985561088:0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.524471:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320972.524546:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561088 00000100:00100000:1.0:1767320972.524553:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561088 00000100:00100000:1.0:1767320972.524615:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5541 00000100:00100000:1.0:1767320972.524624:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985561088:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320972.524703:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320972.524830:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320972.524835:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320972.524838:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320972.524843:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6215 00000100:00080000:3.0:1767320972.524847:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320972.524852:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320972.524872:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f800 x1853163985561216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320972.524937:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985561344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320972.524969:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320972.525000:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561344 00000100:00100000:2.0:1767320972.525005:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561344 00000100:00100000:2.0:1767320972.525052:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5542 00000100:00100000:2.0:1767320972.525060:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985561344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320972.525209:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985561344:12345-0@lo:400:kworker.0 Request processed in 149us (237us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320972.525216:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5542 00000100:00080000:2.0:1767320972.525240:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320972.525263:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985561344:0@lo:400:kworker.0 00000100:00100000:1.0:1767320972.526984:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985561088:12345-0@lo:400:kworker.0 Request processed in 2354us (2504us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320972.526996:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5541 00000100:00100000:2.0:1767320972.527512:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985561216:0@lo:38:kworker.0 00000100:00100000:2.0:1767320972.527572:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320972.527694:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320972.527700:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320972.527703:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320972.527708:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6215 00000100:00080000:3.0:1767320972.527713:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320972.527719:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767320972.527756:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9f80 x1853163985561472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320972.527806:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985561600:0@lo:400:kworker.0 00000100:00100000:3.0:1767320972.527837:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320972.527875:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561600 00000100:00100000:3.0:1767320972.527881:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561600 00000100:00100000:3.0:1767320972.527940:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3587 00000100:00100000:3.0:1767320972.527949:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985561600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320972.527994:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985561600:12345-0@lo:400:kworker.0 Request processed in 47us (155us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320972.528000:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3587 00000100:00080000:2.0:1767320972.528105:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320972.528119:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985561472:0@lo:38:kworker.0 00000100:00100000:2.0:1767320972.528146:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320972.528192:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561216 00000100:00100000:2.0:1767320972.528196:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561216 00000100:00100000:2.0:1767320972.528215:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561472 00000100:00100000:2.0:1767320972.528218:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561472 00000100:00100000:2.0:1767320972.528232:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6327 00000100:00100000:2.0:1767320972.528236:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985561216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320972.528255:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320972.528288:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985561216:12345-0@lo:38:kworker.0 Request processed in 50us (716us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320972.528294:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6327 00000100:00100000:2.0:1767320972.528306:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6328 00000100:00100000:2.0:1767320972.528310:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985561472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320972.528317:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320972.528336:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985561472:12345-0@lo:38:kworker.0 Request processed in 26us (190us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320972.528342:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6328 00000100:00080000:2.0:1767320972.528371:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320972.528376:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320972.528379:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985561472:0@lo:38:kworker.0 00000100:00080000:2.0:1767320972.528387:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320972.528390:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320972.528405:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985561216:0@lo:38:kworker.0 00000100:00100000:1.0:1767320972.529262:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985561728:0@lo:400:kworker.0 00000100:00100000:1.0:1767320972.529308:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320972.529411:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561728 00000100:00100000:2.0:1767320972.529417:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561728 00000100:00100000:2.0:1767320972.529471:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3588 00000100:00100000:2.0:1767320972.529481:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985561728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320972.529519:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985561728:12345-0@lo:400:kworker.0 Request processed in 41us (211us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320972.529526:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3588 00000100:00080000:1.0:1767320972.529596:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320972.529623:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985561728:0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.529855:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985561088:0@lo:400:kworker.0 00000100:00100000:3.0:1767320972.530442:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985561600:0@lo:400:kworker.0 00010000:00010000:2.0:1767320973.148667:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320974.043729:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985561856:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320974.043813:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320974.043863:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320974.043869:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985561984:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320974.043892:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320974.044135:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561856 00000100:00100000:0.0:1767320974.044143:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561856 00000100:00100000:0.0:1767320974.044243:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985561984 00000100:00100000:0.0:1767320974.044246:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985561984 00000100:00100000:0.0:1767320974.044270:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2804 00000100:00100000:0.0:1767320974.044278:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985561856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320974.044307:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320974.044350:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985561856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (537us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320974.044358:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2804 00000100:00100000:0.0:1767320974.044384:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2805 00000100:00100000:0.0:1767320974.044390:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985561984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320974.044406:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320974.044430:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985561984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (537us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320974.044435:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2805 00000100:00080000:3.0:1767320974.044651:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320974.044655:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320974.044694:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985561984:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320974.044708:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985561856:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320974.173592:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320975.195583:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320976.219549:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320977.248206:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320977.578583:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320977.578655:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767320977.578742:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647424 00000100:00100000:1.0:1767320977.578752:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647424 00000100:00100000:1.0:1767320977.578815:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647296 00000100:00100000:1.0:1767320977.578818:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647296 00000100:00100000:1.0:1767320977.578832:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5543 00000100:00100000:1.0:1767320977.578841:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920647424:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767320977.578908:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920647424:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 69us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320977.578915:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5543 00000100:00100000:1.0:1767320977.578931:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6329 00000100:00100000:1.0:1767320977.578935:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920647296:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767320977.578954:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320977.578973:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920647296:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 38us (319us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320977.578979:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6329 00000100:00080000:2.0:1767320977.579412:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320977.628025:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320977.628036:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320977.628039:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320977.628045:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320977.628050:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320977.628058:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6220 00000100:00080000:2.0:1767320977.628064:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320977.628071:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320977.628112:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9c00 x1853163985562112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320977.628128:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320977.628132:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320977.628134:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320977.628137:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6220 00000100:00080000:2.0:1767320977.628140:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320977.628143:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320977.628157:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985562240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320977.628220:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320977.628224:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320977.628226:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320977.628230:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6220 00000100:00080000:2.0:1767320977.628232:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320977.628235:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320977.628257:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7b80 x1853163985562496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320977.628271:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320977.628274:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320977.628276:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320977.628279:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6220 00000100:00080000:2.0:1767320977.628281:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320977.628284:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767320977.628292:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5180 x1853163985562752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320977.628375:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320977.628392:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985562880:0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.628451:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320977.628518:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985562880 00000100:00100000:2.0:1767320977.628525:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985562880 00000100:00100000:2.0:1767320977.628583:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3589 00000100:00100000:2.0:1767320977.628591:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985562880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.628636:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985562880:12345-0@lo:400:kworker.0 Request processed in 46us (184us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320977.628642:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3589 00000100:00080000:2.0:1767320977.628665:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320977.628690:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985562880:0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.628720:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985563008:0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.628741:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320977.628762:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563008 00000100:00100000:2.0:1767320977.628765:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563008 00000100:00100000:2.0:1767320977.628789:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3590 00000100:00100000:2.0:1767320977.628795:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985563008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.628825:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985563008:12345-0@lo:400:kworker.0 Request processed in 30us (83us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320977.628831:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3590 00000100:00080000:2.0:1767320977.628844:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320977.628859:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985563008:0@lo:400:kworker.0 00000100:00100000:3.0:1767320977.629063:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985562368:0@lo:400:kworker.0 00000100:00100000:3.0:1767320977.629096:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320977.629132:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320977.629138:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985562624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320977.629157:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320977.629210:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985562368 00000100:00100000:3.0:1767320977.629214:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985562368 00000100:00100000:3.0:1767320977.629257:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985562624 00000100:00100000:3.0:1767320977.629260:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985562624 00000100:00100000:3.0:1767320977.629279:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5544 00000100:00100000:3.0:1767320977.629285:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985562368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320977.629328:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985562368:12345-0@lo:400:kworker.0 Request processed in 43us (231us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320977.629334:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5544 00000100:00100000:3.0:1767320977.629348:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5545 00000100:00100000:3.0:1767320977.629354:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985562624:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320977.629376:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985562624:12345-0@lo:400:kworker.0 Request processed in 23us (218us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320977.629381:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5545 00000100:00080000:3.0:1767320977.629394:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320977.629411:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985562624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320977.629420:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985562368:0@lo:400:kworker.0 00000100:00100000:0.0:1767320977.631744:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985562112:0@lo:38:kworker.0 00000100:00100000:0.0:1767320977.631825:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320977.631868:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985562240:0@lo:38:kworker.0 00000100:00100000:0.0:1767320977.631894:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320977.631931:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985562496:0@lo:38:kworker.0 00000100:00100000:0.0:1767320977.631952:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320977.631969:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985562752:0@lo:38:kworker.0 00000100:00100000:0.0:1767320977.631987:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320977.632022:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985562112 00000100:00100000:0.0:1767320977.632029:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985562112 00000100:00100000:0.0:1767320977.632056:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985562496 00000100:00100000:0.0:1767320977.632059:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985562496 00000100:00100000:0.0:1767320977.632067:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985562752 00000100:00100000:0.0:1767320977.632069:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985562752 00000100:00100000:0.0:1767320977.632079:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6330 00000100:00100000:0.0:1767320977.632086:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985562112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320977.632107:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320977.632145:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985562112:12345-0@lo:38:kworker.0 Request processed in 58us (319us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320977.632152:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6330 00000100:00100000:0.0:1767320977.632189:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6331 00000100:00100000:0.0:1767320977.632194:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985562496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320977.632204:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320977.632226:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985562496:12345-0@lo:38:kworker.0 Request processed in 30us (272us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320977.632231:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6331 00000100:00100000:0.0:1767320977.632253:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6332 00000100:00100000:0.0:1767320977.632257:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985562752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320977.632265:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320977.632288:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985562752:12345-0@lo:38:kworker.0 Request processed in 30us (300us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320977.632294:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6332 00000100:00080000:0.0:1767320977.632363:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320977.632369:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320977.632373:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985562112:0@lo:38:kworker.0 00000100:00080000:0.0:1767320977.632380:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320977.632383:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320977.632386:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985562496:0@lo:38:kworker.0 00000100:00080000:0.0:1767320977.632392:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320977.632395:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320977.632397:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985562752:0@lo:38:kworker.0 00000100:00100000:3.0:1767320977.633590:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985562240 00000100:00100000:3.0:1767320977.633599:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985562240 00000100:00100000:3.0:1767320977.633643:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2073 00000100:00100000:3.0:1767320977.633650:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985562240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320977.633666:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320977.633725:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985562240:12345-0@lo:38:kworker.0 Request processed in 75us (1829us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320977.633732:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2073 00000100:00080000:0.0:1767320977.634612:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320977.634619:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320977.634624:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985562240:0@lo:38:kworker.0 00010000:00010000:1.0:1767320978.267496:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320979.163545:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320979.163572:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985563136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320979.163649:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320979.164012:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563136 00000100:00100000:1.0:1767320979.164021:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563136 00000100:00100000:1.0:1767320979.164097:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2806 00000100:00100000:1.0:1767320979.164107:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985563136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320979.164135:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320979.164224:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985563136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (573us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320979.164231:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2806 00000100:00100000:1.0:1767320979.164286:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985563136:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320979.164441:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320979.164962:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985563264:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320979.164996:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320979.165184:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563264 00000100:00100000:2.0:1767320979.165190:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563264 00000100:00100000:2.0:1767320979.165285:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2807 00000100:00100000:2.0:1767320979.165293:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985563264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320979.165312:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320979.165518:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985563264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 225us (520us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320979.165526:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2807 00000100:00080000:3.0:1767320979.165567:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320979.165771:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985563264:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320979.292569:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320980.315396:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320981.339954:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320982.363419:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320982.697726:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320982.699344:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647680 00000100:00100000:0.0:1767320982.699357:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647680 00000100:00100000:0.0:1767320982.699445:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5546 00000100:00100000:0.0:1767320982.699457:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920647680:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320982.699550:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920647680:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 96us (1832us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320982.699558:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5546 00000100:00080000:3.0:1767320982.700757:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320982.701851:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320982.701979:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647552 00000100:00100000:2.0:1767320982.701991:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647552 00000100:00100000:2.0:1767320982.702049:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6333 00000100:00100000:2.0:1767320982.702057:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920647552:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320982.702082:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320982.702136:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920647552:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 78us (291us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320982.702143:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6333 00000100:00080000:0.0:1767320982.747504:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320982.747513:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320982.747516:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320982.747520:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320982.747524:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320982.747531:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6225 00000100:00080000:0.0:1767320982.747536:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320982.747542:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320982.747580:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1500 x1853163985563392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320982.747609:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320982.747612:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320982.747614:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320982.747616:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6225 00000100:00080000:0.0:1767320982.747619:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320982.747622:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320982.747635:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3b80 x1853163985563520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320982.747671:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320982.747674:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320982.747676:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320982.747679:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6225 00000100:00080000:0.0:1767320982.747681:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320982.747684:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320982.747696:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60700 x1853163985563776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320982.747697:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320982.747713:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985563648:0@lo:400:kworker.0 00000100:00080000:0.0:1767320982.747717:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320982.747721:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320982.747723:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320982.747726:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6225 00000100:00080000:0.0:1767320982.747728:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320982.747732:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767320982.747742:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a63800 x1853163985564032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320982.747774:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320982.747822:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563648 00000100:00100000:0.0:1767320982.747829:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563648 00000100:00100000:1.0:1767320982.747830:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985563904:0@lo:400:kworker.0 00000100:00100000:1.0:1767320982.747847:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320982.747867:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985564288:0@lo:400:kworker.0 00000100:00100000:1.0:1767320982.747884:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320982.747892:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563904 00000100:00100000:0.0:1767320982.747897:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563904 00000100:00100000:1.0:1767320982.747898:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320982.747904:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985564160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320982.747919:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320982.747928:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5547 00000100:00100000:0.0:1767320982.747939:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985563648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320982.747944:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5548 00000100:00100000:1.0:1767320982.747952:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985563904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320982.748000:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985563904:12345-0@lo:400:kworker.0 Request processed in 48us (150us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320982.748007:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5548 00000100:00100000:2.0:1767320982.748045:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985563392:0@lo:38:kworker.0 00000100:00100000:2.0:1767320982.748075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320982.748102:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985563520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320982.748128:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320982.748177:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320982.748214:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564288 00000100:00100000:3.0:1767320982.748219:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564288 00000100:00100000:3.0:1767320982.748250:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564160 00000100:00100000:3.0:1767320982.748254:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564160 00000100:00100000:3.0:1767320982.748274:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3591 00000100:00100000:3.0:1767320982.748280:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985564288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320982.748320:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985564288:12345-0@lo:400:kworker.0 Request processed in 40us (433us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320982.748326:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3591 00000100:00100000:3.0:1767320982.748339:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3592 00000100:00100000:3.0:1767320982.748345:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985564160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320982.748371:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985564160:12345-0@lo:400:kworker.0 Request processed in 26us (451us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320982.748377:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3592 00000100:00080000:3.0:1767320982.748401:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320982.748404:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320982.748412:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563392 00000100:00100000:3.0:1767320982.748415:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563392 00000100:00100000:3.0:1767320982.748447:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6334 00000100:00100000:3.0:1767320982.748451:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985563392:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320982.748468:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320982.748498:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985563392:12345-0@lo:38:kworker.0 Request processed in 44us (421us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320982.748505:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6334 00000100:00100000:2.0:1767320982.748528:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985563776:0@lo:38:kworker.0 00000100:00100000:3.0:1767320982.748548:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985564160:0@lo:400:kworker.0 00000100:00100000:2.0:1767320982.748556:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320982.748563:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985564288:0@lo:400:kworker.0 00000100:00100000:2.0:1767320982.748588:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985564032:0@lo:38:kworker.0 00000100:00100000:3.0:1767320982.748604:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563520 00000100:00100000:3.0:1767320982.748609:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563520 00000100:00100000:2.0:1767320982.748613:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320982.748635:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2074 00000100:00100000:3.0:1767320982.748641:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985563520:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767320982.748643:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320982.748649:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320982.748653:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985563392:0@lo:38:kworker.0 00010000:00020000:3.0:1767320982.748655:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320982.748680:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985563776 00000100:00100000:2.0:1767320982.748684:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985563776 00000100:00100000:3.0:1767320982.748694:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985563520:12345-0@lo:38:kworker.0 Request processed in 51us (568us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320982.748701:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2074 00000100:00100000:2.0:1767320982.748708:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564032 00000100:00100000:2.0:1767320982.748712:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564032 00000100:00100000:2.0:1767320982.748728:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6335 00000100:00100000:3.0:1767320982.748737:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6336 00000100:00100000:2.0:1767320982.748738:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985563776:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320982.748747:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985564032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320982.748753:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767320982.748759:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320982.748785:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985563776:12345-0@lo:38:kworker.0 Request processed in 50us (229us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320982.748797:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6335 00000100:00100000:3.0:1767320982.748797:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985564032:12345-0@lo:38:kworker.0 Request processed in 47us (182us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320982.748806:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6336 00000100:00080000:2.0:1767320982.748830:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320982.748835:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320982.748838:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985563520:0@lo:38:kworker.0 00000100:00080000:2.0:1767320982.748848:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320982.748852:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320982.748855:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985563776:0@lo:38:kworker.0 00000100:00080000:2.0:1767320982.748862:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320982.748865:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320982.748868:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985564032:0@lo:38:kworker.0 00000100:00100000:0.0:1767320982.749456:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985563648:12345-0@lo:400:kworker.0 Request processed in 1518us (1679us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320982.749464:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5547 00000100:00100000:1.0:1767320982.749471:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985563904:0@lo:400:kworker.0 00000100:00100000:1.0:1767320982.749488:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985563648:0@lo:400:kworker.0 00010000:00010000:0.0:1767320983.387436:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320984.285098:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985564416:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320984.285286:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320984.285396:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320984.285404:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985564544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320984.285432:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320984.286075:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564416 00000100:00100000:2.0:1767320984.286084:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564416 00000100:00100000:2.0:1767320984.286215:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564544 00000100:00100000:2.0:1767320984.286219:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564544 00000100:00100000:1.0:1767320984.286267:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2808 00000100:00100000:1.0:1767320984.286280:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985564416:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320984.286310:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320984.286378:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985564416:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (1091us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320984.286385:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2808 00000100:00080000:3.0:1767320984.286689:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320984.286693:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320984.286735:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985564416:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320984.286849:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2809 00000100:00100000:2.0:1767320984.286858:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985564544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320984.286875:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767320984.286913:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985564544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 57us (1480us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320984.286919:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2809 00000100:00100000:2.0:1767320984.286948:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985564544:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320984.411395:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320985.435472:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320986.462374:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320987.483338:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320987.817555:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320987.817607:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320987.817822:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647936 00000100:00100000:0.0:1767320987.817832:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647936 00000100:00100000:0.0:1767320987.817910:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920647808 00000100:00100000:0.0:1767320987.817913:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920647808 00000100:00100000:0.0:1767320987.817943:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5549 00000100:00100000:0.0:1767320987.817955:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920647936:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320987.818028:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920647936:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (483us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320987.818036:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5549 00000100:00100000:0.0:1767320987.818055:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6337 00000100:00100000:0.0:1767320987.818060:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920647808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767320987.818081:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320987.818106:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920647808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (498us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320987.818123:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6337 00000100:00080000:3.0:1767320987.818578:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320987.871417:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.871429:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320987.871432:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.871437:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.871442:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.871453:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.871458:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.871467:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320987.871529:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c380 x1853163985564672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320987.871559:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.871563:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.871565:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.871569:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.871572:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.871575:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320987.871589:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73fb80 x1853163985564800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320987.871632:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.871636:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.871638:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.871642:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.871644:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.871648:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320987.871659:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73d180 x1853163985565056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320987.871675:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.871678:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.871680:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.871683:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.871686:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.871689:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320987.871701:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73dc00 x1853163985565312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320987.871749:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985565184:0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.871832:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320987.871907:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320987.871913:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985564928:0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.871945:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320987.871971:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985565440:0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.871997:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320987.872014:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320987.872019:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985565568:0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.872039:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320987.872068:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565184 00000100:00100000:1.0:1767320987.872075:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565184 00000100:00100000:1.0:1767320987.872130:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564928 00000100:00100000:1.0:1767320987.872134:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564928 00000100:00100000:1.0:1767320987.872154:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5550 00000100:00100000:1.0:1767320987.872196:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985565184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.872249:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985565184:12345-0@lo:400:kworker.0 Request processed in 87us (416us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320987.872257:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5550 00000100:00100000:1.0:1767320987.872273:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5551 00000100:00100000:1.0:1767320987.872279:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985564928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.872306:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985564928:12345-0@lo:400:kworker.0 Request processed in 27us (360us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320987.872314:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5551 00000100:00100000:1.0:1767320987.872351:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985564928:0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.872361:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985565184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320987.872746:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565440 00000100:00100000:3.0:1767320987.872751:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565440 00000100:00100000:3.0:1767320987.872802:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565568 00000100:00100000:3.0:1767320987.872805:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565568 00000100:00100000:3.0:1767320987.872831:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3593 00000100:00100000:3.0:1767320987.872838:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985565440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320987.872883:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985565440:12345-0@lo:400:kworker.0 Request processed in 45us (884us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320987.872889:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3593 00000100:00100000:3.0:1767320987.872901:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3594 00000100:00100000:3.0:1767320987.872907:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985565568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320987.872945:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985565568:12345-0@lo:400:kworker.0 Request processed in 39us (905us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320987.872951:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3594 00000100:00080000:3.0:1767320987.872970:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320987.872973:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320987.872976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320987.872993:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985565568:0@lo:400:kworker.0 00000100:00100000:3.0:1767320987.873001:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985565440:0@lo:400:kworker.0 00000100:00100000:2.0:1767320987.877415:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985564672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320987.877489:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320987.877551:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985564800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320987.877576:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320987.877607:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985565056:0@lo:38:kworker.0 00000100:00100000:2.0:1767320987.877628:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320987.877652:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985565312:0@lo:38:kworker.0 00000100:00100000:2.0:1767320987.877670:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320987.877712:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564672 00000100:00100000:2.0:1767320987.877720:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564672 00000100:00100000:2.0:1767320987.877751:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565056 00000100:00100000:2.0:1767320987.877754:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565056 00000100:00100000:2.0:1767320987.877778:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565312 00000100:00100000:2.0:1767320987.877781:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565312 00000100:00100000:2.0:1767320987.877798:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6338 00000100:00100000:2.0:1767320987.877804:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985564672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320987.877826:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320987.877862:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985564672:12345-0@lo:38:kworker.0 Request processed in 56us (372us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320987.877868:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6338 00000100:00100000:2.0:1767320987.877882:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6339 00000100:00100000:2.0:1767320987.877886:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985565056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320987.877894:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320987.877910:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985565056:12345-0@lo:38:kworker.0 Request processed in 22us (281us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320987.877915:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6339 00000100:00100000:0.0:1767320987.878205:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6340 00000100:00100000:0.0:1767320987.878211:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985565312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320987.878221:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320987.878248:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985565312:12345-0@lo:38:kworker.0 Request processed in 37us (577us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320987.878254:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6340 00000100:00100000:3.0:1767320987.878538:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985564800 00000100:00100000:3.0:1767320987.878544:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985564800 00000100:00100000:3.0:1767320987.878566:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2075 00000100:00100000:3.0:1767320987.878571:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985564800:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320987.878581:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320987.878609:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985564800:12345-0@lo:38:kworker.0 Request processed in 37us (1031us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320987.878615:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2075 00000100:00080000:0.0:1767320987.881491:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.881504:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.881510:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985564672:0@lo:38:kworker.0 00000100:00080000:0.0:1767320987.881625:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.881629:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.881632:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985564800:0@lo:38:kworker.0 00000100:00080000:0.0:1767320987.881640:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.881644:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.881646:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985565056:0@lo:38:kworker.0 00000100:00080000:0.0:1767320987.881656:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.881659:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.881662:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985565312:0@lo:38:kworker.0 00010000:00010000:2.0:1767320988.508272:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320989.403799:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985565696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320989.403874:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320989.404125:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320989.404133:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985565824:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320989.404160:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320989.404255:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565696 00000100:00100000:0.0:1767320989.404263:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565696 00000100:00100000:0.0:1767320989.404320:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565824 00000100:00100000:0.0:1767320989.404324:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565824 00000100:00100000:0.0:1767320989.404344:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2810 00000100:00100000:0.0:1767320989.404354:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985565696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320989.404385:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320989.404432:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985565696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (557us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320989.404439:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2810 00000100:00100000:0.0:1767320989.404457:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2811 00000100:00100000:0.0:1767320989.404463:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985565824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320989.404478:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767320989.404569:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320989.404574:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320989.404637:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985565696:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320989.404717:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985565824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 253us (555us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320989.404724:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2811 00000100:00100000:3.0:1767320989.404790:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985565824:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767320989.531941:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320990.559969:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320991.579395:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320992.606695:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767320992.937912:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920648064 offset 0 length 520: 4 00000100:00100000:3.0:1767320992.939971:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767320992.942452:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920648192 00000100:00100000:2.0:1767320992.942465:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920648192 00000100:00100000:2.0:1767320992.942526:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6341 00000100:00100000:2.0:1767320992.942534:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920648192:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767320992.942557:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320992.942637:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920648192:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 102us (2674us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320992.942645:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6341 00000100:00100000:1.0:1767320992.945784:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320992.945987:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920648320 00000100:00100000:0.0:1767320992.945997:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920648320 00000100:00100000:0.0:1767320992.946081:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5552 00000100:00100000:0.0:1767320992.946092:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920648320:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320992.946218:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920648320:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 127us (442us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320992.946225:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5552 00000100:00080000:3.0:1767320992.948022:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320992.987266:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320992.987274:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320992.987277:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320992.987280:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320992.987283:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320992.987289:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6236 00000100:00080000:1.0:1767320992.987293:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320992.987307:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320992.987341:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163985565952/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320992.987359:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320992.987361:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320992.987362:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320992.987364:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6236 00000100:00080000:1.0:1767320992.987366:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320992.987368:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320992.987376:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c000 x1853163985566080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320992.987953:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985566208:0@lo:400:kworker.0 00000100:00100000:2.0:1767320992.988023:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320992.988121:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985566208 00000100:00100000:3.0:1767320992.988127:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985566208 00000100:00100000:3.0:1767320992.988180:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5553 00000100:00100000:3.0:1767320992.988186:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985566208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320992.988274:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320967/real 1767320967] req@ffff8ce1a593ad80 x1853163985559424/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320992 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320992.988292:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a593ad80 x1853163985559424/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320992 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320992.988302:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985565952:0@lo:38:kworker.0 00000100:00100000:0.0:1767320992.988322:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320992.988337:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985566080:0@lo:38:kworker.0 00000100:00100000:0.0:1767320992.988351:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320992.988364:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320992.988371:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320992.988373:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985559424:0@lo:250:kworker.0 00000100:00100000:0.0:1767320992.988406:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985566080 00000100:00100000:0.0:1767320992.988410:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985566080 00000100:00100000:0.0:1767320992.988434:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2076 00000100:00100000:0.0:1767320992.988439:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985566080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320992.988456:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320992.988485:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985566080:12345-0@lo:38:kworker.0 Request processed in 45us (133us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320992.988490:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2076 00000100:00100000:0.0:1767320992.988510:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985565952 00000100:00100000:0.0:1767320992.988513:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985565952 00000100:00100000:0.0:1767320992.988524:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6342 00000100:00100000:0.0:1767320992.988527:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985565952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320992.988534:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320992.988548:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985565952:12345-0@lo:38:kworker.0 Request processed in 20us (226us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320992.988552:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6342 00000100:00080000:0.0:1767320992.988571:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320992.988573:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320992.988575:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985565952:0@lo:38:kworker.0 00000100:00080000:0.0:1767320992.988580:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320992.988582:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320992.988584:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985566080:0@lo:38:kworker.0 00000100:00080000:0.0:1767320992.988658:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320992.988705:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320992.988709:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320992.988710:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320992.988713:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6236 00000100:00080000:1.0:1767320992.988716:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320992.988718:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320992.988734:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73dc00 x1853163985566336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320992.988772:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985566464:0@lo:400:kworker.0 00000100:00100000:0.0:1767320992.988789:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320992.988808:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985566464 00000100:00100000:0.0:1767320992.988811:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985566464 00000100:00100000:0.0:1767320992.988828:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5554 00000100:00100000:0.0:1767320992.988831:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985566464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320992.988862:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985566464:12345-0@lo:400:kworker.0 Request processed in 30us (71us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320992.988866:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5554 00000100:00100000:0.0:1767320992.988883:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985566464:0@lo:400:kworker.0 00000100:00080000:0.0:1767320992.988892:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320992.988902:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985566336:0@lo:38:kworker.0 00000100:00100000:0.0:1767320992.988915:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320992.988929:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985566336 00000100:00100000:0.0:1767320992.988931:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985566336 00000100:00100000:0.0:1767320992.988940:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6343 00000100:00100000:0.0:1767320992.988943:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985566336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320992.988950:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320992.988966:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985566336:12345-0@lo:38:kworker.0 Request processed in 22us (50us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320992.988969:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6343 00000100:00080000:0.0:1767320992.988986:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320992.988990:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320992.988992:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985566336:0@lo:38:kworker.0 00000100:00100000:3.0:1767320992.991821:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985566208:12345-0@lo:400:kworker.0 Request processed in 3624us (3796us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320992.992336:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320992.992343:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320992.992347:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320992.992352:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6236 00000100:00080000:1.0:1767320992.992356:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320992.992361:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320992.992408:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73fb80 x1853163985566592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320992.992469:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320992.992472:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320992.992474:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320992.992477:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6215 00000100:00080000:1.0:1767320992.992480:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320992.992484:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320992.992494:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c700 x1853163985566976/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320992.992533:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985566848:0@lo:400:kworker.0 00000100:00100000:1.0:1767320992.992587:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320992.992619:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320992.992624:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985566720:0@lo:400:kworker.0 00000100:00100000:1.0:1767320992.992647:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320992.992673:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985566848 00000100:00100000:1.0:1767320992.992679:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985566848 00000100:00100000:1.0:1767320992.992724:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985566720 00000100:00100000:1.0:1767320992.992727:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985566720 00000100:00100000:1.0:1767320992.992747:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3595 00000100:00100000:1.0:1767320992.992755:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985566848:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320992.992991:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3596 00000100:00100000:0.0:1767320992.993001:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985566720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320992.993056:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985566720:12345-0@lo:400:kworker.0 Request processed in 58us (408us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320992.993062:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3596 00000100:00100000:0.0:1767320992.993098:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985566720:0@lo:400:kworker.0 00000100:00080000:0.0:1767320992.993129:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320992.993133:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320992.993142:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985566592:0@lo:38:kworker.0 00000100:00100000:0.0:1767320992.993195:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320992.993212:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985566976:0@lo:250:kworker.0 00000400:00000100:0.0:1767320992.993230:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985566976 offset 0 length 520: 4 00000100:00100000:0.0:1767320992.993270:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985566592 00000100:00100000:0.0:1767320992.993274:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985566592 00000100:00100000:0.0:1767320992.993294:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6344 00000100:00100000:0.0:1767320992.993299:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985566592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320992.993317:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320992.993349:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985566592:12345-0@lo:38:kworker.0 Request processed in 49us (153us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320992.993354:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6344 00000100:00080000:0.0:1767320992.993381:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320992.993386:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320992.993389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985566592:0@lo:38:kworker.0 00000100:00100000:1.0:1767320992.993778:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985566848:12345-0@lo:400:kworker.0 Request processed in 1015us (1181us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320992.993786:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3595 00000100:00100000:3.0:1767320992.994097:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5553 00000100:00100000:0.0:1767320992.994735:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985566848:0@lo:400:kworker.0 00000100:00100000:2.0:1767320992.995262:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985566208:0@lo:400:kworker.0 00010000:00010000:2.0:1767320993.627814:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320994.524539:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320994.524558:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985567104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320994.524627:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320994.524635:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320994.524645:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985567232:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320994.524672:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320994.524697:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985567104 00000100:00100000:0.0:1767320994.524704:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985567104 00000100:00100000:1.0:1767320994.524730:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985567232 00000100:00100000:1.0:1767320994.524734:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985567232 00000100:00080000:2.0:1767320994.524779:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320994.524784:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320994.524829:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2812 00000100:00100000:1.0:1767320994.524840:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985567232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320994.524867:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320994.524928:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985567232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320994.524935:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2812 00000100:00100000:1.0:1767320994.524952:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2813 00000100:00100000:1.0:1767320994.524958:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985567104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320994.524977:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320994.525012:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985567104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 54us (385us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320994.525019:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2813 00000100:00100000:0.0:1767320994.525043:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985567104:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320994.525147:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985567232:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320994.652449:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767320995.675406:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320996.702245:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320997.723432:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320998.065706:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767320998.065782:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767320998.065985:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920648576 00000100:00100000:0.0:1767320998.065995:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920648576 00000100:00100000:0.0:1767320998.066064:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920648448 00000100:00100000:0.0:1767320998.066068:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920648448 00000100:00100000:0.0:1767320998.066089:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5555 00000100:00100000:0.0:1767320998.066099:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920648576:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767320998.066206:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920648576:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 109us (510us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320998.066214:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5555 00000100:00100000:0.0:1767320998.066233:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6345 00000100:00100000:0.0:1767320998.066238:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920648448:12345-192.168.204.26@tcp:38:kworker.0 00000100:00080000:2.0:1767320998.066436:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:0.0:1767320998.066631:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320998.066707:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920648448:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 467us (925us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320998.066715:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6345 00000100:00080000:1.0:1767320998.107362:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320998.107376:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320998.107379:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320998.107383:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320998.107388:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320998.107397:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6241 00000100:00080000:1.0:1767320998.107402:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320998.107409:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320998.107458:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c000 x1853163985567360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320998.107487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320998.107491:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320998.107492:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320998.107496:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6241 00000100:00080000:1.0:1767320998.107498:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320998.107501:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320998.107516:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163985567488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320998.107557:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320998.107561:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320998.107562:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320998.107565:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6241 00000100:00080000:1.0:1767320998.107568:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320998.107572:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320998.107583:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e680 x1853163985567744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320998.107606:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320998.107609:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320998.107611:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320998.107614:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6241 00000100:00080000:1.0:1767320998.107616:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320998.107619:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767320998.107626:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73fb80 x1853163985568000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320998.107676:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985568128:0@lo:400:kworker.0 00000100:00100000:3.0:1767320998.107776:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985567616:0@lo:400:kworker.0 00000100:00100000:0.0:1767320998.107875:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985567360:0@lo:38:kworker.0 00000100:00100000:1.0:1767320998.107938:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320998.108435:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985567872:0@lo:400:kworker.0 00000100:00100000:3.0:1767320998.111789:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320998.114155:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568128 00000100:00100000:3.0:1767320998.114182:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568128 00000100:00100000:3.0:1767320998.114612:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3597 00000100:00100000:3.0:1767320998.114623:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985568128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320998.114710:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985568128:12345-0@lo:400:kworker.0 Request processed in 89us (6771us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320998.114718:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3597 00000100:00100000:3.0:1767320998.115513:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985568256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320998.115564:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320998.115617:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568256 00000100:00100000:3.0:1767320998.115623:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568256 00000100:00100000:3.0:1767320998.115657:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3598 00000100:00100000:3.0:1767320998.115665:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985568256:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320998.115702:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985568256:12345-0@lo:400:kworker.0 Request processed in 38us (137us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320998.115709:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3598 00000100:00100000:3.0:1767320998.115746:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985568256:0@lo:400:kworker.0 00000100:00080000:3.0:1767320998.115782:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320998.115787:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320998.115820:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985567616 00000100:00100000:3.0:1767320998.115824:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985567616 00000100:00100000:3.0:1767320998.115854:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5556 00000100:00100000:3.0:1767320998.116544:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985567616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320998.116594:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985567616:12345-0@lo:400:kworker.0 Request processed in 56us (4812us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320998.123590:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5556 00000100:00080000:3.0:1767320998.123630:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320998.123683:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985567616:0@lo:400:kworker.0 00000100:00100000:1.0:1767320998.124139:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985568128:0@lo:400:kworker.0 00000100:00100000:1.0:1767320998.124567:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320998.124608:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985567488:0@lo:38:kworker.0 00000100:00100000:1.0:1767320998.124649:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320998.124682:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985567744:0@lo:38:kworker.0 00000100:00100000:1.0:1767320998.124703:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320998.124719:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985568000:0@lo:38:kworker.0 00000100:00100000:1.0:1767320998.124743:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320998.124770:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985567360 00000100:00100000:1.0:1767320998.124776:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985567360 00000100:00100000:1.0:1767320998.124812:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985567744 00000100:00100000:1.0:1767320998.124816:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985567744 00000100:00100000:1.0:1767320998.124831:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568000 00000100:00100000:1.0:1767320998.124834:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568000 00000100:00100000:1.0:1767320998.124850:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6346 00000100:00100000:3.0:1767320998.125068:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6347 00000100:00100000:3.0:1767320998.125074:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985567744:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320998.125096:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320998.125138:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985567744:12345-0@lo:38:kworker.0 Request processed in 63us (434us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320998.125145:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6347 00000100:00100000:3.0:1767320998.125182:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6348 00000100:00100000:3.0:1767320998.125186:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985568000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320998.125196:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320998.125373:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985568000:12345-0@lo:38:kworker.0 Request processed in 184us (628us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320998.125380:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6348 00000100:00100000:3.0:1767320998.125429:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985567488 00000100:00100000:3.0:1767320998.125434:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985567488 00000100:00100000:3.0:1767320998.125456:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2077 00000100:00100000:3.0:1767320998.125460:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985567488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320998.125470:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320998.125494:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985567488:12345-0@lo:38:kworker.0 Request processed in 33us (845us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320998.125500:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2077 00000100:00100000:1.0:1767320998.125585:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985567360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320998.125597:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320998.125626:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985567360:12345-0@lo:38:kworker.0 Request processed in 770us (1061us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320998.125633:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6346 00000100:00080000:1.0:1767320998.125667:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320998.125672:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320998.125676:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985568000:0@lo:38:kworker.0 00000100:00080000:1.0:1767320998.125807:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320998.125812:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320998.125816:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985567360:0@lo:38:kworker.0 00000100:00080000:1.0:1767320998.125828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320998.125831:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320998.125834:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985567488:0@lo:38:kworker.0 00000100:00080000:1.0:1767320998.125841:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320998.125845:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320998.125848:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985567744:0@lo:38:kworker.0 00000100:00100000:2.0:1767320998.126632:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320998.126680:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985567872 00000100:00100000:0.0:1767320998.126687:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985567872 00000100:00100000:0.0:1767320998.126784:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5557 00000100:00100000:0.0:1767320998.126792:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985567872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320998.126850:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985567872:12345-0@lo:400:kworker.0 Request processed in 59us (219us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320998.126869:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5557 00000100:00100000:2.0:1767320998.126968:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985567872:0@lo:400:kworker.0 00000100:00080000:3.0:1767320998.127421:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320998.748518:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320999.643523:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320999.643549:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985568384:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320999.643819:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320999.644037:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985568512:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320999.644066:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320999.644133:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568384 00000100:00100000:0.0:1767320999.644142:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568512 00000100:00100000:1.0:1767320999.644143:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568384 00000100:00100000:0.0:1767320999.644149:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568512 00000100:00100000:0.0:1767320999.644216:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2814 00000100:00100000:0.0:1767320999.644226:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985568512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320999.644254:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767320999.644305:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985568512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320999.644312:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2814 00000100:00100000:3.0:1767320999.644350:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985568512:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320999.646389:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2815 00000100:00100000:1.0:1767320999.646404:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985568384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320999.646430:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767320999.646498:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985568384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (2680us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320999.646506:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2815 00000100:00100000:3.0:1767320999.646587:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985568384:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320999.773953:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321000.795395:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321001.819500:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321002.843562:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321003.177583:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321003.177783:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920648832 00000100:00100000:1.0:1767321003.177795:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920648832 00000100:00100000:1.0:1767321003.177900:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5558 00000100:00100000:1.0:1767321003.177911:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920648832:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321003.178000:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920648832:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (426us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321003.178009:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5558 00000100:00100000:3.0:1767321003.178556:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321003.178703:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920648704 00000100:00100000:0.0:1767321003.178709:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920648704 00000100:00100000:0.0:1767321003.178742:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6349 00000100:00100000:0.0:1767321003.178751:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920648704:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321003.178782:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321003.178842:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920648704:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 92us (288us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321003.178850:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6349 00000100:00080000:2.0:1767321003.179061:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321003.227362:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321003.227373:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321003.227376:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321003.227380:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321003.227384:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321003.227392:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6246 00000100:00080000:1.0:1767321003.227396:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321003.227404:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321003.227441:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633100 x1853163985568640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321003.227457:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321003.227460:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321003.227462:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321003.227466:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6246 00000100:00080000:1.0:1767321003.227468:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321003.227472:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321003.227484:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632680 x1853163985568768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321003.227564:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985568896:0@lo:400:kworker.0 00000100:00100000:2.0:1767321003.227628:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321003.227851:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568896 00000100:00100000:3.0:1767321003.227861:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568896 00000100:00080000:1.0:1767321003.227871:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321003.227876:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321003.227879:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321003.227883:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6246 00000100:00080000:1.0:1767321003.227885:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321003.227890:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321003.227906:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630a80 x1853163985569024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321003.227927:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5559 00000100:00100000:3.0:1767321003.227937:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985568896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321003.228013:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985568640:0@lo:38:kworker.0 00000100:00100000:0.0:1767321003.228044:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321003.228068:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568640 00000100:00100000:0.0:1767321003.228072:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568640 00000100:00100000:0.0:1767321003.228095:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6350 00000100:00100000:0.0:1767321003.228100:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985568640:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321003.228127:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321003.228182:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985568640:12345-0@lo:38:kworker.0 Request processed in 80us (137us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321003.228188:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6350 00000100:00080000:0.0:1767321003.228228:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321003.228232:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321003.228245:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985568640:0@lo:38:kworker.0 00000100:00100000:0.0:1767321003.228252:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985568768:0@lo:38:kworker.0 00000100:00100000:0.0:1767321003.228273:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321003.228294:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985568768 00000100:00100000:0.0:1767321003.228297:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985568768 00000100:00100000:0.0:1767321003.228314:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2078 00000100:00100000:0.0:1767321003.228318:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985568768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321003.228327:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321003.228347:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985568768:12345-0@lo:38:kworker.0 Request processed in 28us (73us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321003.228356:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2078 00000100:00080000:0.0:1767321003.228381:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321003.228384:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321003.228387:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985568768:0@lo:38:kworker.0 00000100:00100000:0.0:1767321003.228393:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985569024:0@lo:38:kworker.0 00000100:00100000:0.0:1767321003.228410:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321003.228423:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569024 00000100:00100000:0.0:1767321003.228426:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569024 00000100:00100000:0.0:1767321003.228440:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6351 00000100:00100000:0.0:1767321003.228444:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985569024:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321003.228453:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321003.228475:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985569024:12345-0@lo:38:kworker.0 Request processed in 29us (63us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321003.228480:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6351 00000100:00080000:0.0:1767321003.228500:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321003.228504:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321003.228507:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985569024:0@lo:38:kworker.0 00000100:00100000:0.0:1767321003.228555:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985569152:0@lo:400:kworker.0 00000100:00100000:0.0:1767321003.228578:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321003.228603:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569152 00000100:00100000:0.0:1767321003.228607:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569152 00000100:00100000:0.0:1767321003.228629:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5560 00000100:00100000:0.0:1767321003.228635:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985569152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321003.228666:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985569152:12345-0@lo:400:kworker.0 Request processed in 32us (88us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321003.228672:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5560 00000100:00100000:0.0:1767321003.228698:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985569152:0@lo:400:kworker.0 00000100:00080000:0.0:1767321003.228721:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321003.232108:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321003.232438:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321003.232442:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321003.232451:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6246 00000100:00080000:1.0:1767321003.232455:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321003.232462:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321003.232504:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632d80 x1853163985569280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321003.232597:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985569536:0@lo:400:kworker.0 00000100:00100000:1.0:1767321003.232664:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321003.232709:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321003.232717:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985569408:0@lo:400:kworker.0 00000100:00100000:3.0:1767321003.232741:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985568896:12345-0@lo:400:kworker.0 Request processed in 4802us (5107us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321003.232746:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321003.232759:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5559 00000100:00100000:1.0:1767321003.233942:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569536 00000100:00100000:1.0:1767321003.233955:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569536 00000100:00100000:1.0:1767321003.234012:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569408 00000100:00100000:1.0:1767321003.234015:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569408 00000100:00100000:1.0:1767321003.234041:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3599 00000100:00100000:1.0:1767321003.234048:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985569536:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321003.236070:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985569280:0@lo:38:kworker.0 00000100:00100000:0.0:1767321003.236460:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321003.236552:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3600 00000100:00100000:0.0:1767321003.236566:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985569408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321003.236615:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985569408:12345-0@lo:400:kworker.0 Request processed in 55us (3871us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321003.236623:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3600 00000100:00100000:0.0:1767321003.236683:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985569408:0@lo:400:kworker.0 00000100:00100000:0.0:1767321003.236697:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985569536:0@lo:400:kworker.0 00000100:00080000:3.0:1767321003.236811:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321003.236816:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321003.236838:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569280 00000100:00100000:3.0:1767321003.236844:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569280 00000100:00100000:3.0:1767321003.236880:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6352 00000100:00100000:3.0:1767321003.236885:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985569280:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321003.236902:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321003.236940:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985569280:12345-0@lo:38:kworker.0 Request processed in 53us (480us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321003.236945:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6352 00000100:00080000:3.0:1767321003.236971:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321003.236976:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321003.236979:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985569280:0@lo:38:kworker.0 00000100:00100000:1.0:1767321003.237449:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985569536:12345-0@lo:400:kworker.0 Request processed in 70us (1454us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321003.237459:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3599 00000100:00100000:2.0:1767321003.238022:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985568896:0@lo:400:kworker.0 00010000:00010000:2.0:1767321003.867742:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321004.763492:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321004.764261:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985569792:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321004.764331:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321004.764372:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985569664:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321004.764394:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321004.769832:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569792 00000100:00100000:2.0:1767321004.769848:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569792 00000100:00100000:2.0:1767321004.769915:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569664 00000100:00100000:2.0:1767321004.769918:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569664 00000100:00100000:2.0:1767321004.769952:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2816 00000100:00100000:2.0:1767321004.769962:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985569792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321004.770000:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321004.770090:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985569792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 129us (5759us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321004.770098:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2816 00000100:00100000:3.0:1767321004.770321:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2817 00000100:00100000:3.0:1767321004.770331:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985569664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321004.770347:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321004.770378:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985569664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 50us (5982us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321004.770383:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2817 00000100:00100000:1.0:1767321004.770487:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985569792:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321004.770502:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985569664:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321004.891924:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321005.922424:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321006.939414:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321007.964462:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321008.298515:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321008.298955:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920648960 00000100:00100000:0.0:1767321008.298965:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920648960 00000100:00100000:0.0:1767321008.299010:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6353 00000100:00100000:0.0:1767321008.299026:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920648960:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321008.299049:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321008.299114:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920648960:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 96us (608us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321008.299121:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6353 00000100:00100000:1.0:1767321008.303939:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321008.304208:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920649088 00000100:00100000:3.0:1767321008.304218:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920649088 00000100:00100000:3.0:1767321008.304292:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5561 00000100:00100000:3.0:1767321008.304302:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920649088:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321008.304383:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920649088:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (454us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321008.304390:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5561 00000100:00080000:0.0:1767321008.306986:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321008.348870:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321008.348880:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321008.348884:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321008.348888:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321008.348892:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321008.348900:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6251 00000100:00080000:0.0:1767321008.348912:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321008.348920:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321008.348962:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99c00 x1853163985569920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321008.348986:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321008.348989:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321008.348991:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321008.348994:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6251 00000100:00080000:0.0:1767321008.348996:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321008.348999:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321008.349011:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9bb80 x1853163985570048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321008.349118:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321008.349123:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321008.349134:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321008.349139:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6251 00000100:00080000:0.0:1767321008.349142:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321008.349147:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321008.349182:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98000 x1853163985570304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321008.349198:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985569920:0@lo:38:kworker.0 00000100:00100000:2.0:1767321008.349267:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985570432:0@lo:400:kworker.0 00000100:00100000:2.0:1767321008.349337:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321008.349398:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321008.349405:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985570176:0@lo:400:kworker.0 00000100:00100000:2.0:1767321008.349428:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321008.349465:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570432 00000100:00100000:2.0:1767321008.349473:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570432 00000100:00100000:3.0:1767321008.349985:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321008.350013:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985570048:0@lo:38:kworker.0 00000100:00100000:3.0:1767321008.350037:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321008.350062:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985570304:0@lo:38:kworker.0 00000100:00100000:3.0:1767321008.350082:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321008.350112:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570176 00000100:00100000:3.0:1767321008.350117:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570176 00000100:00080000:1.0:1767321008.350206:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321008.350231:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985569920 00000100:00100000:1.0:1767321008.350236:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985569920 00000100:00100000:1.0:1767321008.350264:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570304 00000100:00100000:1.0:1767321008.350267:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570304 00000100:00100000:1.0:1767321008.350286:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6354 00000100:00100000:1.0:1767321008.350294:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985569920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321008.350314:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321008.350361:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985569920:12345-0@lo:38:kworker.0 Request processed in 68us (377us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321008.350368:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6354 00000100:00100000:1.0:1767321008.350384:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6355 00000100:00100000:1.0:1767321008.350388:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985570304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321008.350399:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321008.350421:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985570304:12345-0@lo:38:kworker.0 Request processed in 31us (337us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321008.350426:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6355 00000100:00100000:1.0:1767321008.350444:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570048 00000100:00100000:1.0:1767321008.350448:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570048 00000100:00100000:1.0:1767321008.350468:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2079 00000100:00100000:1.0:1767321008.350472:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985570048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321008.350482:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321008.350507:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985570048:12345-0@lo:38:kworker.0 Request processed in 34us (469us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321008.350512:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2079 00000100:00100000:2.0:1767321008.350744:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5562 00000100:00100000:2.0:1767321008.350755:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985570432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321008.350806:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985570432:12345-0@lo:400:kworker.0 Request processed in 54us (1468us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321008.350811:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5562 00000100:00100000:1.0:1767321008.351255:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985570432:0@lo:400:kworker.0 00000100:00080000:1.0:1767321008.351305:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321008.351323:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321008.351327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985570304:0@lo:38:kworker.0 00000100:00080000:1.0:1767321008.351340:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321008.351343:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321008.351346:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985569920:0@lo:38:kworker.0 00000100:00080000:1.0:1767321008.351354:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321008.351357:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321008.351360:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985570048:0@lo:38:kworker.0 00000100:00080000:0.0:1767321008.351458:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321008.351463:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321008.351466:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321008.351471:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6251 00000100:00080000:0.0:1767321008.351475:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321008.351479:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321008.351509:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99180 x1853163985570560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321008.351871:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985570560:0@lo:38:kworker.0 00000100:00100000:1.0:1767321008.351932:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321008.351968:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570560 00000100:00100000:1.0:1767321008.351974:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570560 00000100:00100000:1.0:1767321008.352000:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6356 00000100:00100000:1.0:1767321008.352005:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985570560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321008.352187:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321008.352225:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985570560:12345-0@lo:38:kworker.0 Request processed in 218us (293us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321008.352232:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6356 00000100:00080000:1.0:1767321008.352263:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321008.352267:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321008.352271:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985570560:0@lo:38:kworker.0 00000100:00100000:0.0:1767321008.352484:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985570816:0@lo:400:kworker.0 00000100:00100000:0.0:1767321008.352519:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321008.352550:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321008.352556:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985570688:0@lo:400:kworker.0 00000100:00100000:0.0:1767321008.352576:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321008.352604:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570816 00000100:00100000:0.0:1767321008.352609:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570816 00000100:00100000:0.0:1767321008.352662:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570688 00000100:00100000:0.0:1767321008.352665:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570688 00000100:00100000:0.0:1767321008.352695:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3601 00000100:00100000:0.0:1767321008.352702:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985570816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321008.352751:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985570816:12345-0@lo:400:kworker.0 Request processed in 50us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321008.352757:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3601 00000100:00100000:0.0:1767321008.352772:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3602 00000100:00100000:0.0:1767321008.352778:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985570688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321008.352804:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985570688:12345-0@lo:400:kworker.0 Request processed in 27us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321008.352810:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3602 00000100:00100000:0.0:1767321008.352840:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985570688:0@lo:400:kworker.0 00000100:00100000:0.0:1767321008.352850:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985570816:0@lo:400:kworker.0 00000100:00080000:0.0:1767321008.352866:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321008.352870:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321008.361307:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5563 00000100:00100000:3.0:1767321008.361327:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985570176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321008.363999:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985570176:12345-0@lo:400:kworker.0 Request processed in 2669us (14558us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321008.364015:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5563 00000100:00100000:1.0:1767321008.364061:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985570176:0@lo:400:kworker.0 00010000:00010000:1.0:1767321008.988439:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321009.883343:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321009.883367:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985570944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321009.883435:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321009.883492:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985570944 00000100:00100000:2.0:1767321009.883501:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985570944 00000100:00100000:3.0:1767321009.883710:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321009.883720:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985571072:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321009.883756:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2818 00000100:00100000:2.0:1767321009.883768:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985570944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321009.883799:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321009.883844:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985570944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (408us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321009.883852:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2818 00000100:00100000:3.0:1767321009.883944:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321009.884008:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571072 00000100:00100000:0.0:1767321009.884014:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571072 00000100:00100000:0.0:1767321009.884052:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2819 00000100:00100000:0.0:1767321009.884061:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985571072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321009.884081:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321009.884107:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985570944:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321009.884126:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985571072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (190us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321009.884136:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2819 00000100:00100000:3.0:1767321009.884259:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985571072:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321010.012687:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321011.037419:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321012.059397:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321013.083437:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321013.419295:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321013.419402:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920649344 00000100:00100000:0.0:1767321013.419414:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920649344 00000100:00100000:0.0:1767321013.419503:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5564 00000100:00100000:0.0:1767321013.419515:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920649344:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321013.419615:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920649344:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 102us (333us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321013.419624:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5564 00000100:00080000:1.0:1767321013.420093:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321013.420440:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321013.421121:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920649216 00000100:00100000:3.0:1767321013.421131:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920649216 00000100:00100000:3.0:1767321013.421224:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6357 00000100:00100000:3.0:1767321013.421232:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920649216:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321013.421258:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321013.421327:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920649216:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 94us (889us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321013.421335:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6357 00000100:00080000:1.0:1767321013.468427:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321013.468437:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321013.468440:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321013.468443:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321013.468447:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321013.468457:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6256 00000100:00080000:1.0:1767321013.468461:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321013.468469:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321013.468511:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188630380 x1853163985571200/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321013.468544:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321013.468548:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321013.468549:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321013.468552:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6256 00000100:00080000:1.0:1767321013.468555:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321013.468558:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321013.468573:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631f80 x1853163985571328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321013.468611:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321013.468615:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321013.468617:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321013.468620:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6256 00000100:00080000:1.0:1767321013.468622:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321013.468625:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321013.468649:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632300 x1853163985571584/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321013.468666:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321013.468669:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321013.468671:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321013.468674:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6256 00000100:00080000:1.0:1767321013.468677:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321013.468681:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321013.468690:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188632680 x1853163985571840/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321013.468737:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985572096:0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.468835:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.468908:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321013.468915:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985571968:0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.468952:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.468984:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985572096 00000100:00100000:1.0:1767321013.468991:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985572096 00000100:00100000:1.0:1767321013.469039:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3603 00000100:00100000:1.0:1767321013.469047:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985572096:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.469096:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985572096:12345-0@lo:400:kworker.0 Request processed in 50us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321013.469104:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3603 00000100:00100000:1.0:1767321013.469119:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571968 00000100:00100000:1.0:1767321013.469122:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571968 00000100:00100000:1.0:1767321013.469146:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3604 00000100:00100000:1.0:1767321013.469152:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985571968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.469217:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985571968:12345-0@lo:400:kworker.0 Request processed in 66us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321013.469224:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3604 00000100:00080000:1.0:1767321013.469241:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321013.469246:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321013.469272:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985571968:0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.469283:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985572096:0@lo:400:kworker.0 00000100:00100000:0.0:1767321013.469499:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985571456:0@lo:400:kworker.0 00000100:00100000:0.0:1767321013.469531:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321013.469569:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321013.469575:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985571712:0@lo:400:kworker.0 00000100:00100000:0.0:1767321013.469595:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321013.469648:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571456 00000100:00100000:0.0:1767321013.469654:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571456 00000100:00100000:0.0:1767321013.469686:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571712 00000100:00100000:0.0:1767321013.469689:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571712 00000100:00100000:0.0:1767321013.469707:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5565 00000100:00100000:0.0:1767321013.469713:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985571456:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321013.469749:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985571456:12345-0@lo:400:kworker.0 Request processed in 35us (217us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321013.469755:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5565 00000100:00100000:0.0:1767321013.469769:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5566 00000100:00100000:0.0:1767321013.469774:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985571712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321013.469799:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985571712:12345-0@lo:400:kworker.0 Request processed in 25us (203us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321013.469804:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5566 00000100:00080000:0.0:1767321013.469818:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321013.469833:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985571712:0@lo:400:kworker.0 00000100:00100000:0.0:1767321013.469842:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985571456:0@lo:400:kworker.0 00000100:00100000:3.0:1767321013.469942:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985571200:0@lo:38:kworker.0 00000100:00100000:3.0:1767321013.469971:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321013.470001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985571328:0@lo:38:kworker.0 00000100:00100000:3.0:1767321013.470024:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321013.470047:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985571584:0@lo:38:kworker.0 00000100:00100000:3.0:1767321013.470065:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321013.470082:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985571840:0@lo:38:kworker.0 00000100:00100000:3.0:1767321013.470101:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321013.470132:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571328 00000100:00100000:3.0:1767321013.470136:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571328 00000100:00100000:3.0:1767321013.470158:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2080 00000100:00100000:3.0:1767321013.470187:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985571328:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321013.470215:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321013.470249:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985571328:12345-0@lo:38:kworker.0 Request processed in 85us (224us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321013.470255:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2080 00000100:00100000:3.0:1767321013.470275:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571200 00000100:00100000:3.0:1767321013.470279:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571200 00000100:00100000:3.0:1767321013.470291:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571584 00000100:00100000:3.0:1767321013.470293:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571584 00000100:00100000:3.0:1767321013.470301:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985571840 00000100:00100000:3.0:1767321013.470303:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985571840 00000100:00100000:3.0:1767321013.470313:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6358 00000100:00100000:3.0:1767321013.470318:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985571200:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321013.470327:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321013.470351:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985571200:12345-0@lo:38:kworker.0 Request processed in 33us (380us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321013.470357:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6358 00000100:00100000:3.0:1767321013.470368:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6359 00000100:00100000:3.0:1767321013.470372:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985571584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321013.470380:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321013.470405:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985571584:12345-0@lo:38:kworker.0 Request processed in 31us (339us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321013.470411:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6359 00000100:00100000:3.0:1767321013.470421:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6360 00000100:00100000:3.0:1767321013.470425:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985571840:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321013.470433:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321013.470454:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985571840:12345-0@lo:38:kworker.0 Request processed in 27us (353us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321013.470459:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6360 00000100:00080000:3.0:1767321013.470492:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321013.470497:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321013.470500:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985571200:0@lo:38:kworker.0 00000100:00080000:3.0:1767321013.470507:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321013.470510:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321013.470512:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985571328:0@lo:38:kworker.0 00000100:00080000:3.0:1767321013.470518:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321013.470521:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321013.470523:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985571584:0@lo:38:kworker.0 00000100:00080000:3.0:1767321013.470530:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321013.470533:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321013.470536:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985571840:0@lo:38:kworker.0 00010000:00010000:0.0:1767321014.108937:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321015.003569:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321015.003588:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985572224:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321015.003669:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321015.003763:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985572224 00000100:00100000:0.0:1767321015.003772:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985572224 00000100:00100000:0.0:1767321015.003840:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2820 00000100:00100000:0.0:1767321015.003850:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985572224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321015.003879:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321015.003926:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985572224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (256us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321015.003935:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2820 00000100:00100000:0.0:1767321015.003988:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985572224:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321015.004054:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321015.013390:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985572352:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321015.013468:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321015.013632:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985572352 00000100:00100000:0.0:1767321015.013640:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985572352 00000100:00100000:0.0:1767321015.013705:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2821 00000100:00100000:0.0:1767321015.013715:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985572352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321015.013742:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321015.013796:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985572352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (328us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321015.013803:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2821 00000100:00100000:1.0:1767321015.013892:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985572352:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321015.013954:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767321015.131410:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321016.155381:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321017.179465:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321017.521147:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767321017.521237:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920649472 offset 0 length 520: 4 00000100:00100000:2.0:1767321017.521254:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321017.521372:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920649728 00000100:00100000:1.0:1767321017.521382:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920649728 00000100:00080000:2.0:1767321017.521488:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321017.521589:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920649600 00000100:00100000:1.0:1767321017.521593:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920649600 00000100:00100000:3.0:1767321017.521624:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5567 00000100:00100000:3.0:1767321017.521632:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920649728:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321017.521703:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920649728:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 73us (566us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321017.521709:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5567 00000100:00100000:3.0:1767321017.521721:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6361 00000100:00100000:3.0:1767321017.521724:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920649600:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321017.521749:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321017.521773:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920649600:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (517us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321017.521779:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6361 00000100:00100000:3.0:1767321017.563648:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320992/real 1767320992] req@ffff8ce1af73c700 x1853163985566976/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321017 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321017.563700:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1af73c700 x1853163985566976/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321017 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321017.563720:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321017.563747:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321017.563750:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985566976:0@lo:250:kworker.0 00000100:00080000:1.0:1767321017.564007:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321017.564013:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767321017.564016:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321017.564021:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6241 00000100:00080000:1.0:1767321017.564024:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321017.564030:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321017.564061:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7f800 x1853163985572480/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321017.564087:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321017.564091:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321017.564093:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321017.564096:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6261 00000100:00080000:1.0:1767321017.564098:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321017.564101:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321017.564114:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7d500 x1853163985572608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321017.564127:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321017.564130:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321017.564132:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321017.564135:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6261 00000100:00080000:1.0:1767321017.564138:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321017.564141:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321017.564150:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7fb80 x1853163985572736/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321017.564246:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:1.0:1767321017.564263:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321017.564267:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321017.564269:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321017.564272:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6261 00000100:00080000:1.0:1767321017.564274:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321017.564277:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321017.564289:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7c700 x1853163985572992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321017.564337:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985572480:0@lo:250:kworker.0 00000400:00000100:3.0:1767321017.564382:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985572480 offset 0 length 520: 4 00000100:00100000:3.0:1767321017.564435:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985572608:0@lo:38:kworker.0 00000100:00100000:3.0:1767321017.564463:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321017.564482:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985572736:0@lo:38:kworker.0 00000100:00100000:3.0:1767321017.564503:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321017.564519:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985572992:0@lo:38:kworker.0 00000100:00100000:3.0:1767321017.564537:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321017.564580:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985572736 00000100:00100000:3.0:1767321017.564586:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985572736 00000100:00100000:3.0:1767321017.564619:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2081 00000100:00100000:3.0:1767321017.564626:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985572736:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321017.564645:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321017.564682:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985572736:12345-0@lo:38:kworker.0 Request processed in 55us (176us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321017.564699:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2081 00000100:00100000:3.0:1767321017.564725:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985572608 00000100:00100000:3.0:1767321017.564728:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985572608 00000100:00100000:3.0:1767321017.564745:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985572992 00000100:00100000:3.0:1767321017.564747:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985572992 00000100:00100000:3.0:1767321017.564762:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6362 00000100:00100000:3.0:1767321017.564766:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985572608:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321017.564775:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321017.564800:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985572608:12345-0@lo:38:kworker.0 Request processed in 32us (337us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321017.564806:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6362 00000100:00100000:3.0:1767321017.564817:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6363 00000100:00100000:3.0:1767321017.564820:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985572992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321017.564829:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321017.564862:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985572992:12345-0@lo:38:kworker.0 Request processed in 40us (324us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321017.564871:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6363 00000100:00080000:3.0:1767321017.564898:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321017.564901:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321017.564905:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985572608:0@lo:38:kworker.0 00000100:00080000:3.0:1767321017.564911:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321017.564914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321017.564917:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985572736:0@lo:38:kworker.0 00000100:00080000:3.0:1767321017.564924:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321017.564927:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321017.564929:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985572992:0@lo:38:kworker.0 00000100:00100000:0.0:1767321017.565028:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321017.565038:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985573120:0@lo:400:kworker.0 00000100:00100000:0.0:1767321017.565065:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321017.565098:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985573120 00000100:00100000:0.0:1767321017.565102:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985573120 00000100:00100000:0.0:1767321017.565140:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5568 00000100:00100000:0.0:1767321017.565146:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985573120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321017.565218:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985573120:12345-0@lo:400:kworker.0 Request processed in 71us (151us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321017.565224:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5568 00000100:00080000:0.0:1767321017.565241:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321017.565281:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985573120:0@lo:400:kworker.0 00000100:00100000:2.0:1767321017.570920:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985572864:0@lo:400:kworker.0 00000100:00100000:2.0:1767321017.571265:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321017.572149:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985572864 00000100:00100000:3.0:1767321017.572161:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985572864 00000100:00100000:3.0:1767321017.572353:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5569 00000100:00100000:3.0:1767321017.572363:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985572864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321017.572442:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985572864:12345-0@lo:400:kworker.0 Request processed in 81us (1178us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321017.572450:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5569 00000100:00100000:2.0:1767321017.574472:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985572864:0@lo:400:kworker.0 00000100:00080000:1.0:1767321017.574868:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321017.574874:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321017.574878:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321017.574883:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6261 00000100:00080000:1.0:1767321017.574887:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321017.574893:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321017.574925:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7f480 x1853163985573248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321017.575186:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985573248:0@lo:38:kworker.0 00000100:00100000:2.0:1767321017.575238:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321017.575278:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985573248 00000100:00100000:2.0:1767321017.575284:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985573248 00000100:00100000:2.0:1767321017.575312:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6364 00000100:00100000:2.0:1767321017.575319:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985573248:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321017.575336:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321017.575372:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985573248:12345-0@lo:38:kworker.0 Request processed in 53us (133us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321017.575379:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6364 00000100:00080000:2.0:1767321017.575416:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321017.575420:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321017.575424:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985573248:0@lo:38:kworker.0 00000100:00100000:1.0:1767321017.575606:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985573504:0@lo:400:kworker.0 00000100:00100000:1.0:1767321017.575637:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321017.575668:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321017.575674:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985573376:0@lo:400:kworker.0 00000100:00100000:1.0:1767321017.575707:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321017.575732:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985573504 00000100:00100000:1.0:1767321017.575787:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985573504 00000100:00100000:3.0:1767321017.575885:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985573376 00000100:00100000:3.0:1767321017.575890:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985573376 00000100:00080000:0.0:1767321017.580182:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321017.580195:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321017.580199:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321017.581855:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3605 00000100:00100000:1.0:1767321017.581873:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985573504:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321017.581971:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985573504:12345-0@lo:400:kworker.0 Request processed in 106us (6332us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321017.581978:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3605 00000100:00100000:1.0:1767321017.582037:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985573504:0@lo:400:kworker.0 00000100:00100000:3.0:1767321017.584503:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3606 00000100:00100000:3.0:1767321017.584519:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985573376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321017.584614:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985573376:12345-0@lo:400:kworker.0 Request processed in 102us (8906us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321017.584622:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3606 00000100:00100000:1.0:1767321017.585105:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985573376:0@lo:400:kworker.0 00010000:00010000:2.0:1767321018.204200:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321019.227763:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321020.124055:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321020.124078:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985573632:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321020.124147:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321020.124420:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985573760:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321020.124451:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321020.124535:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985573632 00000100:00100000:3.0:1767321020.124545:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985573632 00000100:00100000:3.0:1767321020.124610:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985573760 00000100:00100000:3.0:1767321020.124614:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985573760 00000100:00100000:3.0:1767321020.124640:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2822 00000100:00100000:3.0:1767321020.124650:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985573632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321020.124682:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321020.124759:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2823 00000100:00100000:0.0:1767321020.124769:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985573760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321020.124790:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767321020.125360:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321020.125366:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321020.132344:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985573632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 7686us (8187us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321020.132932:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985573760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 8161us (8475us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321020.134825:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2822 00000100:00100000:2.0:1767321020.136093:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985573760:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321020.136121:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985573632:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321020.136255:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2823 00010000:00010000:0.0:1767321020.251489:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321021.275395:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321022.299515:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321022.635240:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321022.635296:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321022.635462:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920649984 00000100:00100000:3.0:1767321022.635499:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920649984 00000100:00100000:3.0:1767321022.635577:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920649856 00000100:00100000:3.0:1767321022.635581:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920649856 00000100:00100000:3.0:1767321022.635604:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5570 00000100:00100000:3.0:1767321022.635614:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920649984:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321022.635696:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920649984:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (466us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321022.635705:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5570 00000100:00100000:3.0:1767321022.635726:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6365 00000100:00100000:3.0:1767321022.635732:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920649856:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321022.635754:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321022.635783:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920649856:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 50us (486us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321022.635789:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6365 00000100:00080000:1.0:1767321022.636152:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321022.683997:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321022.684008:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321022.684011:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321022.684015:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321022.684019:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321022.684028:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6265 00000100:00080000:0.0:1767321022.684033:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321022.684040:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321022.684085:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99180 x1853163985573888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321022.684112:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321022.684115:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321022.684117:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321022.684121:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6265 00000100:00080000:0.0:1767321022.684123:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321022.684126:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321022.684140:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9ad80 x1853163985574016/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321022.684213:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321022.684218:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321022.684220:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321022.684223:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6265 00000100:00080000:0.0:1767321022.684226:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321022.684229:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321022.684265:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405f80 x1853163985574272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321022.684292:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321022.684295:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321022.684297:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321022.684301:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6265 00000100:00080000:0.0:1767321022.684303:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321022.684307:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321022.684320:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407800 x1853163985574528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321022.684400:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985574656:0@lo:400:kworker.0 00000100:00100000:1.0:1767321022.684552:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985574144:0@lo:400:kworker.0 00000100:00100000:2.0:1767321022.684607:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985574784:0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.684676:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985574400:0@lo:400:kworker.0 00000100:00100000:1.0:1767321022.687638:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321022.688048:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574144 00000100:00100000:1.0:1767321022.688059:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574144 00000100:00100000:1.0:1767321022.688130:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5571 00000100:00100000:1.0:1767321022.688140:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985574144:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321022.688214:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985574144:12345-0@lo:400:kworker.0 Request processed in 76us (582us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321022.688221:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5571 00000100:00080000:1.0:1767321022.688247:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321022.688415:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321022.688741:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574784 00000100:00100000:2.0:1767321022.688746:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574784 00000100:00100000:2.0:1767321022.688881:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3607 00000100:00100000:2.0:1767321022.688892:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985574784:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321022.688995:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985574784:12345-0@lo:400:kworker.0 Request processed in 106us (579us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321022.689002:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3607 00000100:00100000:2.0:1767321022.689033:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985573888:0@lo:38:kworker.0 00000100:00100000:2.0:1767321022.689065:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321022.689093:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985574016:0@lo:38:kworker.0 00000100:00100000:2.0:1767321022.689117:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321022.689140:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985574272:0@lo:38:kworker.0 00000100:00100000:2.0:1767321022.689161:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321022.689197:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985574528:0@lo:38:kworker.0 00000100:00100000:2.0:1767321022.689220:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321022.689251:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574016 00000100:00100000:2.0:1767321022.689256:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574016 00000100:00100000:2.0:1767321022.689276:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2082 00000100:00100000:1.0:1767321022.689455:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985574144:0@lo:400:kworker.0 00000100:00080000:1.0:1767321022.689489:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321022.689520:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985573888 00000100:00100000:1.0:1767321022.689525:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985573888 00000100:00100000:1.0:1767321022.689543:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574272 00000100:00100000:1.0:1767321022.689546:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574272 00000100:00100000:1.0:1767321022.689558:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574528 00000100:00100000:1.0:1767321022.689560:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574528 00000100:00100000:1.0:1767321022.689574:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6366 00000100:00100000:1.0:1767321022.689578:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985573888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321022.689597:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321022.689634:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985573888:12345-0@lo:38:kworker.0 Request processed in 54us (567us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321022.689640:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6366 00000100:00100000:1.0:1767321022.689651:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6367 00000100:00100000:1.0:1767321022.689654:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985574272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321022.689663:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321022.689687:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985574272:12345-0@lo:38:kworker.0 Request processed in 31us (524us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321022.689692:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6367 00000100:00100000:1.0:1767321022.689702:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6368 00000100:00100000:1.0:1767321022.689706:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985574528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321022.689715:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321022.689736:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985574528:12345-0@lo:38:kworker.0 Request processed in 28us (515us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321022.689741:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6368 00000100:00100000:1.0:1767321022.689778:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985574784:0@lo:400:kworker.0 00000100:00080000:1.0:1767321022.690130:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321022.690136:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321022.690139:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985573888:0@lo:38:kworker.0 00000100:00080000:1.0:1767321022.690149:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321022.690151:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321022.690154:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985574272:0@lo:38:kworker.0 00000100:00080000:1.0:1767321022.690161:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321022.690183:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321022.690186:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985574528:0@lo:38:kworker.0 00000100:00100000:2.0:1767321022.690298:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985574016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321022.690310:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321022.690345:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985574016:12345-0@lo:38:kworker.0 Request processed in 48us (1227us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321022.690351:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2082 00000100:00080000:1.0:1767321022.690570:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321022.690574:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321022.690578:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985574016:0@lo:38:kworker.0 00000100:00100000:3.0:1767321022.690787:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321022.691006:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574400 00000100:00100000:1.0:1767321022.691012:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574400 00000100:00100000:1.0:1767321022.691075:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5572 00000100:00100000:1.0:1767321022.691082:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985574400:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321022.691130:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985574400:12345-0@lo:400:kworker.0 Request processed in 48us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321022.691136:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5572 00000100:00100000:3.0:1767321022.691217:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985574400:0@lo:400:kworker.0 00000100:00080000:2.0:1767321022.691367:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321022.691532:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321022.691656:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574656 00000100:00100000:1.0:1767321022.691661:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574656 00000100:00100000:1.0:1767321022.691746:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3608 00000100:00100000:1.0:1767321022.691753:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985574656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321022.691985:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985574656:0@lo:400:kworker.0 00000100:00100000:1.0:1767321022.692037:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985574656:12345-0@lo:400:kworker.0 Request processed in 284us (504us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321022.692135:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3608 00000100:00080000:2.0:1767321022.692243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321023.324494:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321024.348128:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321025.243445:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985574912:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321025.243452:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985575040:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321025.246722:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321025.246928:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985574912 00000100:00100000:0.0:1767321025.246938:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985574912 00000100:00100000:0.0:1767321025.247147:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2824 00000100:00100000:0.0:1767321025.247159:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985574912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321025.247213:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321025.247267:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985574912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 110us (551us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321025.247274:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2824 00000100:00100000:1.0:1767321025.247456:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985574912:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321025.247653:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321025.247915:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575040 00000100:00100000:1.0:1767321025.247921:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575040 00000100:00080000:2.0:1767321025.248313:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321025.248318:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321025.252364:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2825 00000100:00100000:1.0:1767321025.252385:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985575040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321025.252434:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321025.252541:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985575040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 164us (4887us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321025.252550:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2825 00000100:00100000:3.0:1767321025.252698:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985575040:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321025.371410:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321026.397860:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321027.419427:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321027.759941:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321027.760241:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920650112 00000100:00100000:3.0:1767321027.760251:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920650112 00000100:00100000:2.0:1767321027.760452:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321027.767197:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920650240 00000100:00100000:3.0:1767321027.767214:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920650240 00000100:00100000:3.0:1767321027.767301:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5573 00000100:00100000:3.0:1767321027.767311:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920650240:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321027.767419:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920650240:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 108us (6966us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321027.767427:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5573 00000100:00100000:3.0:1767321027.767446:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6369 00000100:00100000:3.0:1767321027.767450:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920650112:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321027.767470:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321027.767495:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920650112:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (7565us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321027.767501:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6369 00000100:00080000:3.0:1767321027.767653:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321027.806568:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321027.806578:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321027.806581:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321027.806585:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321027.806589:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321027.806597:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6270 00000100:00080000:1.0:1767321027.806602:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321027.806609:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321027.806658:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7ca80 x1853163985575168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321027.806681:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321027.806684:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321027.806687:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321027.806690:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6270 00000100:00080000:1.0:1767321027.806692:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321027.806696:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321027.806710:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985575168:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.806711:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7e300 x1853163985575296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321027.806751:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321027.806756:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321027.806758:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321027.806761:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6270 00000100:00080000:1.0:1767321027.806764:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321027.806768:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321027.806779:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7ea00 x1853163985575552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321027.806799:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767321027.806803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321027.806805:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321027.806808:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321027.806812:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6270 00000100:00080000:1.0:1767321027.806814:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321027.806817:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321027.806829:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db7dc00 x1853163985575808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321027.806881:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985575296:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.806889:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985575680:0@lo:400:kworker.0 00000100:00100000:2.0:1767321027.806916:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.806931:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321027.806949:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985575552:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.806961:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321027.806970:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985575424:0@lo:400:kworker.0 00000100:00100000:2.0:1767321027.806977:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321027.807003:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985575808:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.807010:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321027.807036:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.807044:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985575936:0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.807073:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.807091:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321027.807095:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985576064:0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.807115:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.807142:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575168 00000100:00100000:1.0:1767321027.807150:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575168 00000100:00100000:1.0:1767321027.807200:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575680 00000100:00100000:1.0:1767321027.807203:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575680 00000100:00100000:1.0:1767321027.807229:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575552 00000100:00100000:1.0:1767321027.807232:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575552 00000100:00100000:1.0:1767321027.807239:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575424 00000100:00100000:1.0:1767321027.807241:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575424 00000100:00100000:1.0:1767321027.807252:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575808 00000100:00100000:1.0:1767321027.807254:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575808 00000100:00100000:1.0:1767321027.807265:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5574 00000100:00100000:1.0:1767321027.807272:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985575680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.807318:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985575680:12345-0@lo:400:kworker.0 Request processed in 46us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321027.807324:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5574 00000100:00100000:1.0:1767321027.807339:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5575 00000100:00100000:1.0:1767321027.807344:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985575424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.807380:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985575424:12345-0@lo:400:kworker.0 Request processed in 36us (371us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321027.807387:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5575 00000100:00100000:1.0:1767321027.807398:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6370 00000100:00100000:1.0:1767321027.807402:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985575168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321027.807420:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321027.807446:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985575168:12345-0@lo:38:kworker.0 Request processed in 43us (648us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321027.807452:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6370 00000100:00100000:1.0:1767321027.807462:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6371 00000100:00100000:1.0:1767321027.807465:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985575552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321027.807474:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321027.807493:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985575552:12345-0@lo:38:kworker.0 Request processed in 26us (517us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321027.807499:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6371 00000100:00100000:1.0:1767321027.807509:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6372 00000100:00100000:1.0:1767321027.807512:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985575808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321027.807521:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321027.807541:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985575808:12345-0@lo:38:kworker.0 Request processed in 27us (507us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321027.807546:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6372 00000100:00100000:1.0:1767321027.807562:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575936 00000100:00100000:1.0:1767321027.807565:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575936 00000100:00100000:1.0:1767321027.807589:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576064 00000100:00100000:1.0:1767321027.807592:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576064 00000100:00100000:1.0:1767321027.807609:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3609 00000100:00100000:1.0:1767321027.807614:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985575936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.807645:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985575936:12345-0@lo:400:kworker.0 Request processed in 30us (571us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321027.807651:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3609 00000100:00100000:1.0:1767321027.807661:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3610 00000100:00100000:1.0:1767321027.807666:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985576064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.807691:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985576064:12345-0@lo:400:kworker.0 Request processed in 25us (576us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321027.807696:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3610 00000100:00100000:1.0:1767321027.807748:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985576064:0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.807759:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985575936:0@lo:400:kworker.0 00000100:00080000:2.0:1767321027.807787:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321027.807794:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321027.807801:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985575424:0@lo:400:kworker.0 00000100:00100000:2.0:1767321027.807802:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985575808:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.807817:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985575680:0@lo:400:kworker.0 00000100:00080000:2.0:1767321027.807824:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321027.807829:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321027.807833:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985575168:0@lo:38:kworker.0 00000100:00080000:2.0:1767321027.807856:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321027.807861:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321027.807864:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985575296 00000100:00100000:2.0:1767321027.807865:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985575552:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.807869:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985575296 00000100:00100000:1.0:1767321027.807895:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2083 00000100:00100000:1.0:1767321027.807900:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985575296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321027.807910:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321027.807950:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985575296:12345-0@lo:38:kworker.0 Request processed in 49us (1035us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321027.807957:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2083 00000100:00080000:2.0:1767321027.808054:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321027.808058:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321027.808060:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985575296:0@lo:38:kworker.0 00000100:00080000:3.0:1767321027.808479:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321027.808485:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321027.808487:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321028.443355:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321029.469906:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321030.363575:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321030.363600:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985576192:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321030.363672:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321030.364017:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576192 00000100:00100000:1.0:1767321030.364025:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576192 00000100:00080000:2.0:1767321030.364095:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321030.364101:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2826 00000100:00100000:1.0:1767321030.364111:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985576192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321030.364140:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321030.364228:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985576192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (556us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321030.364237:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2826 00000100:00100000:2.0:1767321030.364344:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985576192:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321030.366911:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985576320:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321030.366995:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321030.367975:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576320 00000100:00100000:1.0:1767321030.367986:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576320 00000100:00100000:1.0:1767321030.368065:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2827 00000100:00100000:1.0:1767321030.368075:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985576320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321030.368107:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767321030.368427:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321030.368777:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985576320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 702us (1780us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321030.368787:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2827 00000100:00100000:2.0:1767321030.368866:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985576320:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321030.491401:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321031.517871:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321032.539452:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321032.877620:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321032.877672:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321032.877865:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920650496 00000100:00100000:0.0:1767321032.877875:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920650496 00000100:00100000:0.0:1767321032.877938:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920650368 00000100:00100000:0.0:1767321032.877942:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920650368 00000100:00100000:0.0:1767321032.877960:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5576 00000100:00100000:0.0:1767321032.877971:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920650496:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321032.878049:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920650496:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (438us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321032.878057:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5576 00000100:00100000:0.0:1767321032.878077:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6373 00000100:00100000:0.0:1767321032.878081:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920650368:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321032.878099:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321032.878123:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920650368:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (451us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321032.878129:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6373 00000100:00080000:3.0:1767321032.878977:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321032.924097:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321032.924109:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321032.924112:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321032.924123:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321032.924128:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321032.924137:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6276 00000100:00080000:0.0:1767321032.924142:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321032.924150:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321032.924210:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407480 x1853163985576448/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321032.924238:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321032.924241:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321032.924243:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321032.924246:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6276 00000100:00080000:0.0:1767321032.924249:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321032.924253:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321032.924266:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404000 x1853163985576576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321032.924310:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985576448:0@lo:38:kworker.0 00000100:00080000:0.0:1767321032.924312:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321032.924319:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321032.924321:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321032.924325:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6276 00000100:00080000:0.0:1767321032.924327:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321032.924339:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321032.924353:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985576832/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321032.924373:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321032.924377:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321032.924379:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767321032.924384:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321032.924386:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6276 00000100:00080000:0.0:1767321032.924390:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321032.924393:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321032.924414:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404380 x1853163985577088/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321032.924478:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576448 00000100:00100000:0.0:1767321032.924484:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576448 00000100:00100000:0.0:1767321032.924511:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6374 00000100:00100000:0.0:1767321032.924516:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985576448:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321032.924531:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321032.924570:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985576448:12345-0@lo:38:kworker.0 Request processed in 52us (189us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321032.924577:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6374 00000100:00100000:0.0:1767321032.924610:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985577344:0@lo:400:kworker.0 00000100:00100000:3.0:1767321032.924621:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985576960:0@lo:400:kworker.0 00000100:00100000:0.0:1767321032.924639:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321032.924675:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321032.924681:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321032.924709:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321032.924716:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985576704:0@lo:400:kworker.0 00000100:00100000:3.0:1767321032.924738:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321032.924770:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576960 00000100:00100000:3.0:1767321032.924774:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576960 00000100:00100000:3.0:1767321032.924813:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576704 00000100:00100000:3.0:1767321032.924817:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576704 00000100:00100000:3.0:1767321032.924836:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5577 00000100:00100000:3.0:1767321032.924842:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985576960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321032.924882:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985576960:12345-0@lo:400:kworker.0 Request processed in 40us (209us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321032.924888:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5577 00000100:00100000:3.0:1767321032.924902:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5578 00000100:00100000:3.0:1767321032.924908:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985576704:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321032.924936:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985576704:12345-0@lo:400:kworker.0 Request processed in 28us (197us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321032.924942:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5578 00000100:00080000:3.0:1767321032.924958:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321032.924989:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985576704:0@lo:400:kworker.0 00000100:00100000:3.0:1767321032.925000:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985576960:0@lo:400:kworker.0 00000100:00100000:1.0:1767321032.925067:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577344 00000100:00100000:1.0:1767321032.925072:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577344 00000100:00100000:1.0:1767321032.925126:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3611 00000100:00100000:1.0:1767321032.925135:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985577344:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321032.925190:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321032.925198:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321032.925200:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985577344:12345-0@lo:400:kworker.0 Request processed in 64us (557us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321032.925204:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985576448:0@lo:38:kworker.0 00000100:00100000:1.0:1767321032.925213:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3611 00000100:00100000:2.0:1767321032.925217:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985576576:0@lo:38:kworker.0 00000100:00100000:2.0:1767321032.925248:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321032.925606:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985577216:0@lo:400:kworker.0 00000100:00100000:1.0:1767321032.925697:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576576 00000100:00100000:1.0:1767321032.925702:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576576 00000100:00100000:1.0:1767321032.925727:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2084 00000100:00100000:1.0:1767321032.925731:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985576576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321032.925745:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767321032.925970:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321032.925976:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321032.926016:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985577344:0@lo:400:kworker.0 00000100:00100000:0.0:1767321032.926045:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577216 00000100:00100000:0.0:1767321032.926048:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577216 00000100:00100000:0.0:1767321032.926149:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3612 00000100:00100000:0.0:1767321032.926157:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985577216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321032.926220:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985577216:12345-0@lo:400:kworker.0 Request processed in 64us (246us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321032.926227:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3612 00000100:00100000:0.0:1767321032.926254:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985577216:0@lo:400:kworker.0 00000100:00080000:3.0:1767321032.926832:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321032.932181:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985576576:12345-0@lo:38:kworker.0 Request processed in 6421us (6906us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321032.932199:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2084 00000100:00080000:2.0:1767321032.942907:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321032.942920:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321032.942926:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985576576:0@lo:38:kworker.0 00000100:00100000:2.0:1767321032.942965:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985576832:0@lo:38:kworker.0 00000100:00100000:2.0:1767321032.943022:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321032.943120:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985577088:0@lo:38:kworker.0 00000100:00100000:2.0:1767321032.943142:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321032.950515:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985576832 00000100:00100000:1.0:1767321032.950532:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985576832 00000100:00100000:3.0:1767321032.950694:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577088 00000100:00100000:3.0:1767321032.950700:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577088 00000100:00100000:0.0:1767321032.950760:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6375 00000100:00100000:3.0:1767321032.950772:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6376 00000100:00100000:3.0:1767321032.950782:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985577088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321032.950808:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321032.950817:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985576832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321032.950831:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321032.951067:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985576832:12345-0@lo:38:kworker.0 Request processed in 297us (8043us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767321032.951729:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767321032.951732:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985577088:12345-0@lo:38:kworker.0 Request processed in 946us (8584us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767321032.951741:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321032.951748:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985576832:0@lo:38:kworker.0 00000100:00080000:1.0:1767321032.951765:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321032.951768:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321032.951771:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985577088:0@lo:38:kworker.0 00000100:00100000:0.0:1767321032.952762:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6375 00000100:00100000:3.0:1767321032.955365:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6376 00010000:00010000:1.0:1767321033.565686:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321034.587607:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321035.483777:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321035.483802:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985577472:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321035.483878:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321035.483893:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321035.484314:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577472 00000100:00100000:0.0:1767321035.484323:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577472 00000100:00100000:0.0:1767321035.484390:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2828 00000100:00100000:0.0:1767321035.484399:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb1450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985577472:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321035.484428:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767321035.484548:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321035.484789:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb1450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985577472:12345-0@lo:13:osp-pre-0-1.0 Request processed in 390us (910us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321035.484797:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2828 00000100:00100000:2.0:1767321035.484903:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985577600:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321035.484946:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321035.485012:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577600 00000100:00100000:0.0:1767321035.485017:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577600 00000100:00100000:3.0:1767321035.485061:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2829 00000100:00080000:2.0:1767321035.485090:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321035.485116:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985577600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321035.485223:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321035.485338:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985577600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 270us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321035.485405:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985577472:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321035.485480:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985577600:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321035.487003:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2829 00010000:00010000:2.0:1767321035.611939:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321036.635836:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321037.659679:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321037.995404:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321037.995463:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321037.995708:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920650752 00000100:00100000:2.0:1767321037.995719:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920650752 00000100:00100000:2.0:1767321037.995794:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920650624 00000100:00100000:2.0:1767321037.995799:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920650624 00000100:00100000:2.0:1767321037.995826:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5579 00000100:00100000:2.0:1767321037.995838:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920650752:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321037.995919:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920650752:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (525us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321037.995927:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5579 00000100:00100000:2.0:1767321037.995947:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6377 00000100:00100000:2.0:1767321037.995951:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920650624:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321037.995970:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321037.995994:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920650624:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 41us (532us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321037.996000:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6377 00000100:00080000:3.0:1767321037.996265:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321038.044198:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321038.044207:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321038.044211:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321038.044215:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321038.044219:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321038.044228:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321038.044233:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321038.044240:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321038.044281:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5500 x1853163985577728/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321038.044308:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321038.044312:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321038.044314:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321038.044318:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321038.044320:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321038.044324:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321038.044336:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7b80 x1853163985577856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321038.044370:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321038.044373:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321038.044375:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321038.044379:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321038.044381:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321038.044384:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321038.044395:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4700 x1853163985578112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321038.044421:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321038.044424:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321038.044426:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321038.044429:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321038.044431:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321038.044435:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321038.044444:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff7480 x1853163985578368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321038.044511:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985578496:0@lo:400:kworker.0 00000100:00100000:2.0:1767321038.044575:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321038.044735:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985578624:0@lo:400:kworker.0 00000100:00100000:1.0:1767321038.044765:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321038.044800:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985578496 00000100:00100000:1.0:1767321038.044808:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985578496 00000100:00100000:1.0:1767321038.044871:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985578624 00000100:00100000:1.0:1767321038.044874:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985578624 00000100:00100000:1.0:1767321038.044896:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3613 00000100:00100000:1.0:1767321038.044904:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985578496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321038.044948:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985578496:12345-0@lo:400:kworker.0 Request processed in 44us (373us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321038.044955:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3613 00000100:00100000:1.0:1767321038.044972:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3614 00000100:00100000:1.0:1767321038.044979:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985578624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321038.045008:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985578624:12345-0@lo:400:kworker.0 Request processed in 30us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321038.045015:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3614 00000100:00080000:1.0:1767321038.045036:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321038.045039:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321038.045055:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985577728:0@lo:38:kworker.0 00000100:00100000:1.0:1767321038.045082:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321038.045108:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577728 00000100:00100000:1.0:1767321038.045111:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577728 00000100:00100000:1.0:1767321038.045132:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6378 00000100:00100000:1.0:1767321038.045136:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985577728:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321038.045153:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321038.045225:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985577728:12345-0@lo:38:kworker.0 Request processed in 86us (141us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321038.045230:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6378 00000100:00100000:3.0:1767321038.045622:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985577984:0@lo:400:kworker.0 00000100:00100000:3.0:1767321038.045656:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321038.046050:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985578496:0@lo:400:kworker.0 00000100:00100000:0.0:1767321038.046139:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985578240:0@lo:400:kworker.0 00000100:00100000:0.0:1767321038.046205:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321038.046291:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577984 00000100:00100000:1.0:1767321038.046299:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577984 00000100:00100000:1.0:1767321038.046355:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985578240 00000100:00100000:1.0:1767321038.046358:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985578240 00000100:00100000:1.0:1767321038.046379:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5580 00000100:00100000:1.0:1767321038.046387:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985577984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321038.046440:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985577984:12345-0@lo:400:kworker.0 Request processed in 53us (782us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321038.046446:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5580 00000100:00100000:1.0:1767321038.046460:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5581 00000100:00100000:1.0:1767321038.046728:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985578240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321038.046765:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985578240:12345-0@lo:400:kworker.0 Request processed in 299us (560us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321038.046774:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5581 00000100:00100000:3.0:1767321038.047037:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985578624:0@lo:400:kworker.0 00000100:00080000:0.0:1767321038.047108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321038.047135:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985578240:0@lo:400:kworker.0 00000100:00100000:3.0:1767321038.047291:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985577984:0@lo:400:kworker.0 00000100:00080000:1.0:1767321038.047349:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321038.047354:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321038.047358:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985577728:0@lo:38:kworker.0 00000100:00100000:1.0:1767321038.047365:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985577856:0@lo:38:kworker.0 00000100:00100000:1.0:1767321038.047399:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321038.050573:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985578112:0@lo:38:kworker.0 00000100:00100000:1.0:1767321038.050639:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321038.050670:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985578368:0@lo:38:kworker.0 00000100:00100000:1.0:1767321038.050688:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321038.051499:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985578112 00000100:00100000:0.0:1767321038.051509:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985578112 00000100:00100000:0.0:1767321038.051550:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985578368 00000100:00100000:0.0:1767321038.051553:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985578368 00000100:00100000:0.0:1767321038.051576:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6379 00000100:00100000:0.0:1767321038.051583:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985578112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321038.051618:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321038.051673:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985578112:12345-0@lo:38:kworker.0 Request processed in 88us (1033us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321038.051680:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6379 00000100:00100000:0.0:1767321038.051697:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6380 00000100:00100000:0.0:1767321038.051700:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985578368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321038.051708:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321038.051729:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985578368:12345-0@lo:38:kworker.0 Request processed in 27us (1040us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321038.051733:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6380 00000100:00080000:1.0:1767321038.052278:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321038.052286:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321038.052289:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985578112:0@lo:38:kworker.0 00000100:00080000:1.0:1767321038.052299:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321038.052302:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321038.052305:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985578368:0@lo:38:kworker.0 00000100:00100000:2.0:1767321038.052394:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985577856 00000100:00100000:2.0:1767321038.052400:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985577856 00000100:00100000:2.0:1767321038.052785:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2085 00000100:00100000:2.0:1767321038.052791:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985577856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321038.052806:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321038.052869:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985577856:12345-0@lo:38:kworker.0 Request processed in 76us (5469us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321038.052876:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2085 00000100:00080000:1.0:1767321038.052934:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321038.052939:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321038.052942:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985577856:0@lo:38:kworker.0 00010000:00010000:0.0:1767321038.683992:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321039.711228:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321040.603497:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321040.606668:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985578880:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321040.606747:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321040.606899:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985578752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321040.606922:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321040.613320:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985578880 00000100:00100000:3.0:1767321040.613337:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985578880 00000100:00100000:3.0:1767321040.613423:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985578752 00000100:00100000:3.0:1767321040.613427:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985578752 00000100:00100000:3.0:1767321040.613450:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2830 00000100:00100000:3.0:1767321040.613459:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985578880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321040.613503:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321040.613582:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985578880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 123us (6833us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321040.613589:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2830 00000100:00100000:3.0:1767321040.613606:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2831 00000100:00100000:3.0:1767321040.613612:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985578752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321040.613630:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321040.613658:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985578752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 41us (6729us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321040.613663:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2831 00000100:00100000:3.0:1767321040.613712:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985578880:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321040.613726:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985578752:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321040.615347:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321040.615356:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321040.731614:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321041.755954:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321042.139304:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321017/real 1767321017] req@ffff8ce19db7f800 x1853163985572480/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321042.139335:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19db7f800 x1853163985572480/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321042.139355:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321042.139386:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321042.139390:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985572480:0@lo:250:kworker.0 00000100:00080000:0.0:1767321042.139491:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321042.139496:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321042.139499:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321042.139503:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6265 00000100:00080000:0.0:1767321042.139507:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321042.139512:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321042.139544:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5406300 x1853163985579008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321042.139568:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321042.139572:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321042.139574:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321042.139578:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6286 00000100:00080000:0.0:1767321042.139580:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321042.139583:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321042.139598:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404a80 x1853163985579136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321042.139613:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321042.139616:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321042.139623:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321042.139626:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6286 00000100:00080000:0.0:1767321042.139628:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321042.139631:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321042.139640:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404380 x1853163985579264/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321042.139680:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321042.139683:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321042.139685:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321042.139688:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6286 00000100:00080000:0.0:1767321042.139691:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321042.139694:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321042.139703:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404000 x1853163985579520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321042.139716:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321042.139720:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321042.139721:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321042.139724:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6286 00000100:00080000:0.0:1767321042.139727:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321042.139729:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321042.139736:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5406680 x1853163985579776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321042.139800:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985579904:0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.139856:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321042.139913:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985579648:0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.139916:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321042.139925:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985580032:0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.139962:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321042.139967:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321042.139995:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985579904 00000100:00100000:0.0:1767321042.140003:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985579904 00000100:00100000:1.0:1767321042.140021:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321042.140030:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985579392:0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.140053:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580032 00000100:00100000:0.0:1767321042.140057:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580032 00000100:00100000:1.0:1767321042.140059:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321042.140095:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985579648 00000100:00100000:0.0:1767321042.140100:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3615 00000100:00100000:1.0:1767321042.140100:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985579648 00000100:00100000:0.0:1767321042.140122:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985579904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.140146:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985579392 00000100:00100000:1.0:1767321042.140150:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985579392 00000100:00100000:1.0:1767321042.140200:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5582 00000100:00100000:0.0:1767321042.140207:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985579904:12345-0@lo:400:kworker.0 Request processed in 87us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321042.140211:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985579648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.140219:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3615 00000100:00100000:0.0:1767321042.140238:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3616 00000100:00100000:0.0:1767321042.140246:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985580032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.140274:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985579648:12345-0@lo:400:kworker.0 Request processed in 66us (308us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321042.140281:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985580032:12345-0@lo:400:kworker.0 Request processed in 35us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321042.140286:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5582 00000100:00100000:0.0:1767321042.140291:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3616 00000100:00100000:1.0:1767321042.140308:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5583 00000100:00100000:1.0:1767321042.140316:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985579392:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321042.140340:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321042.140347:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321042.140349:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321042.140354:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985579392:12345-0@lo:400:kworker.0 Request processed in 37us (295us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321042.140362:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5583 00000100:00100000:0.0:1767321042.140375:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985580032:0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.140390:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985579904:0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.140400:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985579392:0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.140416:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985579648:0@lo:400:kworker.0 00000100:00100000:2.0:1767321042.140542:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985579008:0@lo:250:kworker.0 00000400:00000100:2.0:1767321042.140564:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985579008 offset 0 length 520: 4 00000100:00100000:2.0:1767321042.140582:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985579136:0@lo:38:kworker.0 00000100:00100000:2.0:1767321042.140600:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321042.140622:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985579264:0@lo:38:kworker.0 00000100:00100000:2.0:1767321042.140640:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321042.140660:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985579520:0@lo:38:kworker.0 00000100:00100000:2.0:1767321042.140677:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321042.140691:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985579776:0@lo:38:kworker.0 00000100:00100000:2.0:1767321042.140705:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321042.140726:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985579136 00000100:00100000:2.0:1767321042.140729:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985579136 00000100:00100000:2.0:1767321042.140744:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985579520 00000100:00100000:2.0:1767321042.140745:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985579520 00000100:00100000:2.0:1767321042.140756:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985579776 00000100:00100000:2.0:1767321042.140757:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985579776 00000100:00100000:2.0:1767321042.140766:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6381 00000100:00100000:2.0:1767321042.140770:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985579136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321042.140786:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321042.140812:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985579136:12345-0@lo:38:kworker.0 Request processed in 41us (211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321042.140818:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6381 00000100:00100000:2.0:1767321042.140828:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6382 00000100:00100000:2.0:1767321042.140831:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985579520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321042.140839:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321042.140854:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985579520:12345-0@lo:38:kworker.0 Request processed in 22us (177us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321042.140860:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6382 00000100:00100000:2.0:1767321042.140867:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6383 00000100:00100000:2.0:1767321042.140870:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985579776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321042.140879:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321042.140895:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985579776:12345-0@lo:38:kworker.0 Request processed in 23us (189us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321042.140901:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6383 00000100:00080000:2.0:1767321042.140931:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321042.140935:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321042.140938:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985579136:0@lo:38:kworker.0 00000100:00080000:2.0:1767321042.140945:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321042.140947:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321042.140950:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985579520:0@lo:38:kworker.0 00000100:00080000:2.0:1767321042.140956:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321042.140959:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321042.140961:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985579776:0@lo:38:kworker.0 00000100:00100000:0.0:1767321042.141020:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985579264 00000100:00100000:0.0:1767321042.141024:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985579264 00000100:00100000:0.0:1767321042.141052:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2086 00000100:00100000:0.0:1767321042.141056:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985579264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321042.141066:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321042.141113:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985579264:12345-0@lo:38:kworker.0 Request processed in 55us (472us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321042.141118:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2086 00000100:00080000:2.0:1767321042.142046:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321042.142055:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321042.142059:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985579264:0@lo:38:kworker.0 00010000:00010000:2.0:1767321042.780078:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321043.114880:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321043.115221:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651136 00000100:00100000:3.0:1767321043.115232:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651136 00000100:00100000:3.0:1767321043.115336:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5584 00000100:00100000:3.0:1767321043.115346:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920651136:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321043.115435:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920651136:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 90us (565us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321043.115443:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5584 00000400:00000100:3.0:1767321043.115532:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920650880 offset 0 length 520: 4 00000100:00080000:0.0:1767321043.115996:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321043.116112:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321043.126075:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651008 00000100:00100000:2.0:1767321043.126100:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651008 00000100:00100000:2.0:1767321043.126182:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6384 00000100:00100000:2.0:1767321043.126192:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920651008:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321043.126217:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321043.127205:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920651008:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 972us (11053us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321043.127216:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6384 00010000:00010000:2.0:1767321043.803732:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321044.830284:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321045.723530:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985580160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321045.723610:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321045.723654:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580160 00000100:00100000:3.0:1767321045.723660:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321045.723665:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580160 00000100:00100000:3.0:1767321045.723670:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985580288:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321045.723697:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321045.723735:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580288 00000100:00100000:3.0:1767321045.723739:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580288 00000100:00100000:1.0:1767321045.723762:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2832 00000100:00080000:0.0:1767321045.723804:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321045.723809:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321045.723882:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985580160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321045.723919:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321045.723981:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985580160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 106us (372us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321045.724062:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985580160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321045.730707:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2833 00000100:00100000:3.0:1767321045.730730:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985580288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321045.730765:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321045.730853:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985580288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 134us (7155us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321045.730861:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2833 00000100:00100000:2.0:1767321045.730965:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985580288:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321045.735426:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2832 00010000:00010000:1.0:1767321045.854199:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321046.877262:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321047.259817:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321047.259828:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321047.259831:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321047.259835:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321047.259839:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321047.259848:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6290 00000100:00080000:0.0:1767321047.259853:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321047.259860:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321047.259905:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405880 x1853163985580416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321047.259929:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321047.259932:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321047.259934:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321047.259938:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6290 00000100:00080000:0.0:1767321047.259940:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321047.259944:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321047.259956:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405180 x1853163985580544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321047.259986:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321047.259990:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321047.259991:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321047.259995:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6290 00000100:00080000:0.0:1767321047.259998:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321047.260002:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321047.260023:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405f80 x1853163985580800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321047.260040:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321047.260044:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321047.260046:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321047.260049:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6290 00000100:00080000:0.0:1767321047.260051:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321047.260054:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321047.260064:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404380 x1853163985581056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321047.260104:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985580672:0@lo:400:kworker.0 00000100:00100000:0.0:1767321047.260183:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321047.260241:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321047.260247:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985580928:0@lo:400:kworker.0 00000100:00100000:0.0:1767321047.260269:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321047.260312:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580672 00000100:00100000:0.0:1767321047.260320:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580672 00000100:00100000:0.0:1767321047.260366:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580928 00000100:00100000:0.0:1767321047.260369:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580928 00000100:00100000:0.0:1767321047.260392:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5585 00000100:00100000:0.0:1767321047.260400:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985580672:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321047.260445:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985580672:12345-0@lo:400:kworker.0 Request processed in 46us (262us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321047.260452:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5585 00000100:00100000:0.0:1767321047.260469:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5586 00000100:00100000:0.0:1767321047.260475:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985580928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321047.260516:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985580928:12345-0@lo:400:kworker.0 Request processed in 41us (246us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321047.260523:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5586 00000100:00080000:0.0:1767321047.260540:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321047.260569:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985580928:0@lo:400:kworker.0 00000100:00100000:0.0:1767321047.260579:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985580672:0@lo:400:kworker.0 00000100:00100000:2.0:1767321047.264176:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985580416:0@lo:38:kworker.0 00000100:00100000:2.0:1767321047.264241:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321047.264284:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985580544:0@lo:38:kworker.0 00000100:00100000:2.0:1767321047.264310:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321047.264345:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985580800:0@lo:38:kworker.0 00000100:00100000:2.0:1767321047.264367:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321047.264373:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580544 00000100:00100000:3.0:1767321047.264382:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580544 00000100:00100000:2.0:1767321047.264395:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985581056:0@lo:38:kworker.0 00000100:00100000:2.0:1767321047.264416:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321047.264420:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2087 00000100:00100000:3.0:1767321047.264428:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985580544:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321047.264446:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985581184:0@lo:400:kworker.0 00000100:00100000:2.0:1767321047.264471:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321047.264488:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580416 00000100:00100000:2.0:1767321047.264489:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321047.264493:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580416 00000100:00100000:2.0:1767321047.264495:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985581312:0@lo:400:kworker.0 00000100:00100000:1.0:1767321047.264516:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985580800 00000100:00100000:2.0:1767321047.264516:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321047.264518:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985580800 00000100:00100000:1.0:1767321047.264526:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581056 00000100:00100000:1.0:1767321047.264528:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581056 00000100:00100000:1.0:1767321047.264538:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6385 00000100:00100000:2.0:1767321047.264543:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6386 00000100:00100000:2.0:1767321047.264547:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985580800:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321047.264554:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985580416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321047.264568:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321047.264639:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581184 00000100:00100000:0.0:1767321047.264644:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581184 00000100:00100000:0.0:1767321047.264688:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581312 00000100:00100000:0.0:1767321047.264692:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581312 00000100:00100000:0.0:1767321047.264712:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3617 00000100:00100000:0.0:1767321047.264719:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985581184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321047.264767:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985581184:12345-0@lo:400:kworker.0 Request processed in 48us (294us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321047.264774:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3617 00000100:00100000:0.0:1767321047.264792:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3618 00000100:00100000:0.0:1767321047.264798:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985581312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321047.264826:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985581312:12345-0@lo:400:kworker.0 Request processed in 29us (309us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321047.264832:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3618 00000100:00100000:0.0:1767321047.264855:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6387 00000100:00100000:0.0:1767321047.264859:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985581056:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321047.264909:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985580800:12345-0@lo:38:kworker.0 Request processed in 359us (539us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321047.264915:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6386 00000100:00080000:2.0:1767321047.264957:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321047.264962:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321047.264966:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985580800:0@lo:38:kworker.0 00000100:00080000:2.0:1767321047.265039:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321047.265043:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321047.265070:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985581312:0@lo:400:kworker.0 00000100:00100000:2.0:1767321047.265079:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985581184:0@lo:400:kworker.0 00010000:00020000:1.0:1767321047.265139:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321047.265271:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985580416:12345-0@lo:38:kworker.0 Request processed in 726us (1028us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321047.265278:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6385 00010000:00020000:3.0:1767321047.265344:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321047.265375:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985580544:12345-0@lo:38:kworker.0 Request processed in 948us (1064us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321047.265381:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2087 00010000:00020000:0.0:1767321047.265920:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321047.265957:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985581056:12345-0@lo:38:kworker.0 Request processed in 1096us (1540us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321047.265964:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6387 00000100:00080000:2.0:1767321047.266208:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321047.266214:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321047.266217:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985580416:0@lo:38:kworker.0 00000100:00080000:2.0:1767321047.266228:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321047.266231:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321047.266234:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985580544:0@lo:38:kworker.0 00000100:00080000:2.0:1767321047.266315:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321047.266319:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321047.266322:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985581056:0@lo:38:kworker.0 00010000:00010000:2.0:1767321047.899396:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321048.238205:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321048.238296:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321048.238423:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651392 00000100:00100000:1.0:1767321048.238436:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651392 00000100:00100000:1.0:1767321048.238512:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651264 00000100:00100000:1.0:1767321048.238516:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651264 00000100:00080000:2.0:1767321048.238526:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321048.238547:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5587 00000100:00100000:1.0:1767321048.238560:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920651392:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321048.238567:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6388 00000100:00100000:2.0:1767321048.238575:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920651264:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321048.238601:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321048.238660:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920651392:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 102us (492us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321048.238668:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5587 00000100:00100000:2.0:1767321048.240395:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920651264:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1806us (2085us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321048.240410:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6388 00010000:00010000:2.0:1767321048.923384:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321049.947484:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321050.843697:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321050.843723:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985581440:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321050.843842:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321050.843852:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a552ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985581568:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321050.843889:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321050.844161:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321050.844235:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581440 00000100:00100000:3.0:1767321050.844243:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581440 00000100:00100000:3.0:1767321050.844356:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581568 00000100:00100000:1.0:1767321050.844358:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2834 00000100:00100000:3.0:1767321050.844359:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581568 00000100:00080000:2.0:1767321050.844603:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321050.844607:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321050.844661:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985581440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321050.844692:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321050.844696:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2835 00000100:00100000:3.0:1767321050.844703:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985581568:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321050.844719:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321050.844752:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985581568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 50us (591us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321050.844759:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2835 00000100:00100000:3.0:1767321050.844819:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a552ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985581568:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321050.844839:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985581440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 472us (950us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321050.844845:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2834 00000100:00100000:3.0:1767321050.844851:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985581440:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321050.971571:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321051.995428:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321052.379387:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321052.379398:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321052.379401:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321052.379406:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321052.379411:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321052.379419:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6295 00000100:00080000:0.0:1767321052.379424:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321052.379432:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321052.379487:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407b80 x1853163985581696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321052.379504:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321052.379508:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321052.379510:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321052.379513:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6295 00000100:00080000:0.0:1767321052.379515:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321052.379518:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321052.379534:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985581824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321052.379574:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985581696:0@lo:38:kworker.0 00000100:00080000:0.0:1767321052.379577:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321052.379580:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321052.379582:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321052.379585:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6295 00000100:00080000:0.0:1767321052.379587:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321052.379590:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321052.379601:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405180 x1853163985582080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321052.379615:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321052.379618:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321052.379620:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321052.379623:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6295 00000100:00080000:0.0:1767321052.379624:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321052.379627:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321052.379634:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5406680 x1853163985582336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321052.379640:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321052.379688:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581696 00000100:00100000:0.0:1767321052.379697:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581696 00000100:00100000:3.0:1767321052.379724:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985581952:0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.379736:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6389 00000100:00100000:0.0:1767321052.379743:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985581696:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321052.379753:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767321052.379764:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321052.379814:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321052.379819:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321052.379822:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985581696:0@lo:38:kworker.0 00000100:00100000:2.0:1767321052.379829:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985581824:0@lo:38:kworker.0 00000100:00100000:1.0:1767321052.379834:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985582208:0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.379870:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985581696:12345-0@lo:38:kworker.0 Request processed in 125us (229us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321052.379877:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6389 00000100:00100000:0.0:1767321052.379893:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581952 00000100:00100000:0.0:1767321052.379897:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581952 00000100:00100000:0.0:1767321052.379944:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5588 00000100:00100000:0.0:1767321052.379950:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985581952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.379990:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985581952:12345-0@lo:400:kworker.0 Request processed in 40us (235us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321052.379996:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5588 00000100:00080000:0.0:1767321052.380016:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321052.380052:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985582464:0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.380081:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321052.380104:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321052.380110:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985582592:0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.380131:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321052.380159:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582464 00000100:00100000:0.0:1767321052.380186:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582464 00000100:00100000:0.0:1767321052.380216:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582592 00000100:00100000:0.0:1767321052.380219:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582592 00000100:00100000:0.0:1767321052.380237:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3619 00000100:00100000:0.0:1767321052.380242:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985582464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.380278:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985582464:12345-0@lo:400:kworker.0 Request processed in 36us (196us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321052.380284:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3619 00000100:00100000:0.0:1767321052.380296:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3620 00000100:00100000:0.0:1767321052.380301:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985582592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.380327:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985582592:12345-0@lo:400:kworker.0 Request processed in 26us (195us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321052.380332:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3620 00000100:00080000:0.0:1767321052.380355:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321052.380358:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321052.380376:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985582592:0@lo:400:kworker.0 00000100:00100000:0.0:1767321052.380385:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985582464:0@lo:400:kworker.0 00000100:00100000:3.0:1767321052.380682:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985581952:0@lo:400:kworker.0 00000100:00100000:2.0:1767321052.380774:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321052.380905:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985582080:0@lo:38:kworker.0 00000100:00100000:2.0:1767321052.380943:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321052.380974:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985582336:0@lo:38:kworker.0 00000100:00100000:2.0:1767321052.380996:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321052.381664:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985581824 00000100:00100000:3.0:1767321052.381673:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985581824 00000100:00100000:3.0:1767321052.381713:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2088 00000100:00100000:3.0:1767321052.381720:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985581824:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321052.381739:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321052.381787:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a552ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985581824:12345-0@lo:38:kworker.0 Request processed in 66us (1013us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321052.381794:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2088 00000100:00080000:3.0:1767321052.381828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321052.381832:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321052.381835:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985581824:0@lo:38:kworker.0 00000100:00100000:0.0:1767321052.382718:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582080 00000100:00100000:0.0:1767321052.382731:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582080 00000100:00100000:0.0:1767321052.382783:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582336 00000100:00100000:0.0:1767321052.382787:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582336 00000100:00100000:0.0:1767321052.382813:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6390 00000100:00100000:0.0:1767321052.382820:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985582080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321052.382840:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321052.382928:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985582080:12345-0@lo:38:kworker.0 Request processed in 106us (1983us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321052.382944:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6390 00000100:00100000:0.0:1767321052.382962:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6391 00000100:00100000:0.0:1767321052.382966:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985582336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321052.382976:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321052.382999:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985582336:12345-0@lo:38:kworker.0 Request processed in 32us (2002us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321052.383006:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6391 00000100:00100000:1.0:1767321052.383492:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321052.383587:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582208 00000100:00100000:2.0:1767321052.383593:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582208 00000100:00100000:2.0:1767321052.383645:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5589 00000100:00100000:2.0:1767321052.383662:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985582208:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321052.383714:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985582208:12345-0@lo:400:kworker.0 Request processed in 63us (224us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321052.383722:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5589 00000100:00100000:1.0:1767321052.383956:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985582208:0@lo:400:kworker.0 00000100:00080000:0.0:1767321052.384067:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321052.390107:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321052.390121:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321052.390127:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985582080:0@lo:38:kworker.0 00000100:00080000:3.0:1767321052.390142:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321052.391279:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321052.391288:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985582336:0@lo:38:kworker.0 00010000:00010000:2.0:1767321053.019461:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321053.355620:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321053.355671:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321053.355710:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651520 00000100:00100000:0.0:1767321053.355719:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651520 00000100:00100000:0.0:1767321053.355765:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651648 00000100:00100000:0.0:1767321053.355768:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651648 00000100:00100000:0.0:1767321053.355810:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5590 00000100:00100000:0.0:1767321053.355821:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920651648:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321053.355920:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920651648:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 101us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321053.355928:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5590 00000100:00100000:0.0:1767321053.356647:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6392 00000100:00100000:0.0:1767321053.356655:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920651520:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321053.356678:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321053.356720:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af1bf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920651520:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 64us (1110us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321053.356727:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6392 00000100:00080000:0.0:1767321053.356753:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321054.044188:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321055.067635:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321055.963477:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321055.963502:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985582720:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321055.963581:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321055.963795:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582720 00000100:00100000:2.0:1767321055.963802:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582720 00000100:00100000:2.0:1767321055.963889:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2836 00000100:00100000:2.0:1767321055.963900:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985582720:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321055.963930:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767321055.964047:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321055.964146:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985582720:12345-0@lo:13:osp-pre-0-1.0 Request processed in 249us (563us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321055.964153:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2836 00000100:00100000:0.0:1767321055.964261:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985582720:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321055.965279:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321055.965295:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985582848:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321055.965347:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321055.965516:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582848 00000100:00100000:2.0:1767321055.965523:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582848 00000100:00100000:2.0:1767321055.965619:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2837 00000100:00100000:2.0:1767321055.965626:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985582848:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321055.965646:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321055.965683:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985582848:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (336us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321055.965687:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2837 00000100:00100000:1.0:1767321055.965781:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985582848:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321055.966104:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321056.091529:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321057.115398:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321057.500635:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321057.500643:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321057.500646:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321057.500650:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321057.500654:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321057.500661:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6300 00000100:00080000:2.0:1767321057.500666:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321057.500672:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321057.500707:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9880 x1853163985582976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321057.500735:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321057.500739:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321057.500741:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321057.500744:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6300 00000100:00080000:2.0:1767321057.500747:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321057.500750:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321057.500767:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9f80 x1853163985583104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321057.500798:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321057.500802:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321057.500804:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321057.500807:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6300 00000100:00080000:2.0:1767321057.500810:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321057.500813:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321057.500835:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb800 x1853163985583360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321057.500852:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321057.500855:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321057.500856:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321057.500859:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6300 00000100:00080000:2.0:1767321057.500862:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321057.500869:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321057.500880:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5180 x1853163985583616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321057.501002:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985583232:0@lo:400:kworker.0 00000100:00100000:1.0:1767321057.501133:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985583744:0@lo:400:kworker.0 00000100:00100000:0.0:1767321057.501246:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985582976:0@lo:38:kworker.0 00000100:00100000:2.0:1767321057.509011:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985583488:0@lo:400:kworker.0 00000100:00100000:1.0:1767321057.510279:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321057.510355:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321057.510363:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321057.510368:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985583872:0@lo:400:kworker.0 00000100:00100000:1.0:1767321057.510402:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321057.510428:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985583232 00000100:00100000:3.0:1767321057.510438:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985583232 00000100:00100000:3.0:1767321057.510502:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5591 00000100:00100000:3.0:1767321057.510772:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985583232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321057.510841:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985583232:12345-0@lo:400:kworker.0 Request processed in 331us (478us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321057.510848:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5591 00000100:00100000:1.0:1767321057.510871:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985583744 00000100:00100000:1.0:1767321057.510877:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985583744 00000100:00080000:3.0:1767321057.510879:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321057.510906:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321057.510934:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985583872 00000100:00100000:3.0:1767321057.510939:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985583232:0@lo:400:kworker.0 00000100:00100000:1.0:1767321057.510940:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985583872 00000100:00080000:3.0:1767321057.510969:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321057.510979:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3621 00000100:00100000:1.0:1767321057.510988:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985583744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321057.511025:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985583744:12345-0@lo:400:kworker.0 Request processed in 38us (750us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321057.511031:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3621 00000100:00100000:1.0:1767321057.511043:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3622 00000100:00100000:1.0:1767321057.511049:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985583872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321057.511076:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985583872:12345-0@lo:400:kworker.0 Request processed in 28us (674us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321057.511082:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3622 00000100:00100000:1.0:1767321057.511110:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985583872:0@lo:400:kworker.0 00000100:00100000:1.0:1767321057.511121:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985583744:0@lo:400:kworker.0 00000100:00100000:0.0:1767321057.511302:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321057.511329:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985582976 00000100:00100000:0.0:1767321057.511333:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985582976 00000100:00100000:0.0:1767321057.511358:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6393 00000100:00100000:0.0:1767321057.511364:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985582976:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321057.511372:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767321057.511384:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321057.511422:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985582976:12345-0@lo:38:kworker.0 Request processed in 57us (121us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321057.511430:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6393 00000100:00100000:0.0:1767321057.511442:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985583488 00000100:00080000:1.0:1767321057.511504:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321057.511509:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321057.511512:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985582976:0@lo:38:kworker.0 00000100:00100000:1.0:1767321057.511521:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985583104:0@lo:38:kworker.0 00000100:00100000:0.0:1767321057.511631:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985583488 00000100:00100000:0.0:1767321057.511664:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5592 00000100:00100000:0.0:1767321057.511670:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985583488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321057.511710:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985583488:12345-0@lo:400:kworker.0 Request processed in 40us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321057.511716:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5592 00000100:00080000:0.0:1767321057.511735:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321057.512559:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321057.512605:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985583360:0@lo:38:kworker.0 00000100:00100000:0.0:1767321057.512634:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321057.512653:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985583360 00000100:00100000:0.0:1767321057.512657:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985583360 00000100:00100000:0.0:1767321057.512677:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6394 00000100:00100000:0.0:1767321057.512681:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985583360:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321057.512697:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321057.512722:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985583360:12345-0@lo:38:kworker.0 Request processed in 39us (87us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321057.512727:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6394 00000100:00080000:0.0:1767321057.512752:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321057.512756:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321057.512759:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985583360:0@lo:38:kworker.0 00000100:00100000:0.0:1767321057.512765:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985583616:0@lo:38:kworker.0 00000100:00100000:0.0:1767321057.512783:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321057.512797:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985583616 00000100:00100000:0.0:1767321057.512800:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985583616 00000100:00100000:0.0:1767321057.512814:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6395 00000100:00100000:0.0:1767321057.512829:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985583616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321057.512846:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321057.512870:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985583616:12345-0@lo:38:kworker.0 Request processed in 51us (86us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321057.512876:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6395 00000100:00080000:0.0:1767321057.512896:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321057.512900:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321057.512903:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985583616:0@lo:38:kworker.0 00000100:00100000:1.0:1767321057.513269:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985583104 00000100:00100000:1.0:1767321057.513274:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985583104 00000100:00100000:1.0:1767321057.513298:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2089 00000100:00100000:1.0:1767321057.513302:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985583104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321057.513312:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321057.513344:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985583104:12345-0@lo:38:kworker.0 Request processed in 40us (787us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321057.513350:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2089 00000100:00080000:0.0:1767321057.513422:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321057.513426:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321057.513429:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985583104:0@lo:38:kworker.0 00000100:00100000:2.0:1767321057.519725:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985583488:0@lo:400:kworker.0 00010000:00010000:1.0:1767321058.139349:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321058.474562:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321058.474708:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321058.474846:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651904 00000100:00100000:0.0:1767321058.474859:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651904 00000100:00100000:0.0:1767321058.474946:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920651776 00000100:00100000:0.0:1767321058.474950:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920651776 00000100:00100000:0.0:1767321058.474976:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5593 00000100:00100000:0.0:1767321058.474986:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920651904:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767321058.475060:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321058.475092:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920651904:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (539us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321058.475103:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5593 00000100:00100000:0.0:1767321058.475126:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6396 00000100:00100000:0.0:1767321058.475131:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920651776:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321058.475149:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321058.475234:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920651776:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 100us (525us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321058.475240:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6396 00010000:00010000:2.0:1767321059.163539:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321060.187406:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321061.084504:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321061.084527:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985584000:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321061.084594:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321061.084730:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321061.084740:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985584128:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321061.084781:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321061.084843:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584000 00000100:00100000:1.0:1767321061.084852:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584000 00000100:00100000:1.0:1767321061.084908:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584128 00000100:00100000:1.0:1767321061.084912:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584128 00000100:00100000:1.0:1767321061.084935:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2838 00000100:00100000:1.0:1767321061.084944:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985584000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321061.084973:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321061.085017:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985584000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (423us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321061.085024:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2838 00000100:00100000:1.0:1767321061.085041:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2839 00000100:00100000:1.0:1767321061.085047:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985584128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321061.085062:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321061.085088:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985584128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (307us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321061.085094:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2839 00000100:00080000:1.0:1767321061.085128:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321061.085133:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321061.085864:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985584000:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321061.089701:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985584128:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321061.211959:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321062.236834:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321062.619328:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321062.619478:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321062.619482:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321062.619486:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321062.619490:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321062.619498:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6305 00000100:00080000:3.0:1767321062.619502:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321062.619510:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321062.619551:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9880 x1853163985584256/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321062.619579:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321062.619583:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321062.619584:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321062.619587:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6305 00000100:00080000:3.0:1767321062.619590:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321062.619593:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321062.619608:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdaa00 x1853163985584384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321062.619632:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985584256:0@lo:38:kworker.0 00000100:00100000:2.0:1767321062.619683:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:3.0:1767321062.619701:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767321062.619704:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321062.619705:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321062.619707:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321062.619710:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6305 00000100:00080000:3.0:1767321062.619712:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321062.619717:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321062.619737:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1500 x1853163985584640/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321062.619756:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584256 00000100:00100000:2.0:1767321062.619778:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985584768:0@lo:400:kworker.0 00000100:00100000:2.0:1767321062.619806:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321062.619843:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985584384:0@lo:38:kworker.0 00000100:00100000:2.0:1767321062.619843:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985584512:0@lo:400:kworker.0 00000100:00100000:2.0:1767321062.619864:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321062.619867:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321062.619887:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584384 00000100:00100000:2.0:1767321062.619891:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584768 00000100:00100000:0.0:1767321062.619894:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584384 00000100:00100000:2.0:1767321062.619894:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584768 00000100:00100000:0.0:1767321062.619926:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2090 00000100:00100000:2.0:1767321062.619930:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584512 00000100:00100000:0.0:1767321062.619933:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985584384:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321062.619933:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584512 00000100:00100000:2.0:1767321062.619952:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5594 00010000:00020000:0.0:1767321062.619953:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321062.619957:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985584768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321062.619987:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985584384:12345-0@lo:38:kworker.0 Request processed in 53us (118us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321062.619991:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985584768:12345-0@lo:400:kworker.0 Request processed in 35us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321062.619994:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2090 00000100:00100000:2.0:1767321062.619996:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5594 00000100:00100000:2.0:1767321062.620009:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5595 00000100:00100000:2.0:1767321062.620014:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985584512:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321062.620039:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767321062.620039:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985584512:12345-0@lo:400:kworker.0 Request processed in 25us (174us total) trans 0 rc 0/0 00000100:00080000:0.0:1767321062.620043:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321062.620045:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5595 00000100:00100000:0.0:1767321062.620047:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985584384:0@lo:38:kworker.0 00000100:00080000:2.0:1767321062.620058:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321062.620064:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985584640:0@lo:38:kworker.0 00000100:00100000:2.0:1767321062.620092:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985584768:0@lo:400:kworker.0 00000100:00100000:2.0:1767321062.620107:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985584512:0@lo:400:kworker.0 00000100:00100000:1.0:1767321062.620160:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584256 00000100:00100000:1.0:1767321062.620212:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6397 00000100:00100000:1.0:1767321062.620217:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985584256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321062.620227:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321062.620255:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985584256:12345-0@lo:38:kworker.0 Request processed in 36us (551us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321062.620261:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6397 00000100:00100000:0.0:1767321062.620463:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321062.620545:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321062.620549:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321062.620553:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985584256:0@lo:38:kworker.0 00000100:00080000:3.0:1767321062.621635:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321062.621642:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321062.621645:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321062.621650:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6305 00000100:00080000:3.0:1767321062.621654:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321062.621659:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321062.621686:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4000 x1853163985584896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321062.621794:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985585024:0@lo:400:kworker.0 00000100:00100000:3.0:1767321062.621839:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321062.621869:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321062.621875:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985585152:0@lo:400:kworker.0 00000100:00100000:3.0:1767321062.621895:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321062.621922:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985584896:0@lo:38:kworker.0 00000100:00100000:3.0:1767321062.621945:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321062.622009:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985585024 00000100:00100000:0.0:1767321062.622016:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985585024 00000100:00100000:0.0:1767321062.622051:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985585152 00000100:00100000:0.0:1767321062.622055:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985585152 00000100:00100000:0.0:1767321062.622073:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3623 00000100:00100000:0.0:1767321062.622081:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985585024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321062.622128:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985585024:12345-0@lo:400:kworker.0 Request processed in 48us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321062.622135:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3623 00000100:00100000:0.0:1767321062.622149:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3624 00000100:00100000:0.0:1767321062.622155:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985585152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321062.622199:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985585152:12345-0@lo:400:kworker.0 Request processed in 44us (302us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321062.622206:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3624 00000100:00100000:2.0:1767321062.623254:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584640 00000100:00100000:2.0:1767321062.623262:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584640 00000100:00100000:3.0:1767321062.623305:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985585152:0@lo:400:kworker.0 00000100:00100000:3.0:1767321062.623334:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985585024:0@lo:400:kworker.0 00000100:00100000:2.0:1767321062.623345:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6398 00000100:00100000:2.0:1767321062.623351:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985584640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321062.623371:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321062.623420:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985584640:12345-0@lo:38:kworker.0 Request processed in 68us (2957us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321062.623427:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6398 00000100:00100000:2.0:1767321062.623441:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985584896 00000100:00100000:2.0:1767321062.623444:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985584896 00000100:00100000:0.0:1767321062.623476:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6399 00000100:00100000:0.0:1767321062.623481:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985584896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321062.623491:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767321062.623493:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321062.623497:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321062.623501:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985584640:0@lo:38:kworker.0 00000100:00100000:0.0:1767321062.623978:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985584896:12345-0@lo:38:kworker.0 Request processed in 494us (2030us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321062.623985:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6399 00000100:00080000:3.0:1767321062.624051:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321062.624056:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321062.624059:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985584896:0@lo:38:kworker.0 00010000:00010000:0.0:1767321063.260689:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321063.597283:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321063.597380:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321063.610116:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920652032 00000100:00100000:2.0:1767321063.610130:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920652032 00000100:00100000:2.0:1767321063.610207:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920652160 00000100:00100000:2.0:1767321063.610211:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920652160 00000100:00100000:2.0:1767321063.610281:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5596 00000100:00100000:2.0:1767321063.610292:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920652160:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321063.610377:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920652160:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (12996us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321063.610385:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5596 00000100:00100000:2.0:1767321063.610405:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6400 00000100:00100000:2.0:1767321063.610410:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920652032:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321063.610430:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321063.610456:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920652032:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (13184us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321063.610462:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6400 00000100:00080000:0.0:1767321063.611203:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321064.284027:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321065.309204:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321066.203434:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985585280:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321066.203468:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985585408:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321066.203500:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321066.203531:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321066.203622:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985585280 00000100:00100000:0.0:1767321066.203629:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985585280 00000100:00100000:0.0:1767321066.203686:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985585408 00000100:00100000:0.0:1767321066.203689:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985585408 00000100:00100000:0.0:1767321066.203711:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2840 00000100:00100000:0.0:1767321066.203717:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985585280:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321066.203734:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2841 00002000:00000024:0.0:1767321066.203741:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321066.203747:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985585408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321066.203769:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321066.203783:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985585280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 64us (281us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321066.203791:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2840 00000100:00100000:1.0:1767321066.203824:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985585408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (294us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321066.203833:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2841 00000100:00100000:0.0:1767321066.203850:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985585280:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321066.203974:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321066.203978:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321066.204000:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985585408:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321066.332909:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321067.355408:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321067.690395:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767321067.690466:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920652288 offset 0 length 520: 4 00000100:00100000:2.0:1767321067.692247:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920652544 00000100:00100000:2.0:1767321067.692261:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920652544 00000100:00100000:2.0:1767321067.692574:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5597 00000100:00100000:2.0:1767321067.692586:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920652544:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321067.692686:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920652544:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 102us (2300us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321067.692695:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5597 00000100:00080000:0.0:1767321067.692939:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321067.701884:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321067.702053:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920652416 00000100:00100000:3.0:1767321067.702064:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920652416 00000100:00100000:3.0:1767321067.702117:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6401 00000100:00100000:3.0:1767321067.702125:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920652416:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321067.702146:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321067.702262:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920652416:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 129us (391us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321067.702270:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6401 00000100:00100000:0.0:1767321067.739534:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321042/real 1767321042] req@ffff8ce1a5406300 x1853163985579008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321067 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321067.739569:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a5406300 x1853163985579008/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321067 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767321067.739592:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321067.742011:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767321067.742023:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985579008:0@lo:250:kworker.0 00000100:00080000:3.0:1767321067.744709:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321067.744719:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321067.744724:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321067.744731:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6290 00000100:00080000:3.0:1767321067.744736:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321067.744742:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321067.744778:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc0a80 x1853163985585536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321067.744845:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985585536:0@lo:250:kworker.0 00000400:00000100:1.0:1767321067.745522:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985585536 offset 0 length 520: 4 00000100:00080000:3.0:1767321067.745854:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321067.745859:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321067.745872:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321067.745877:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6310 00000100:00080000:3.0:1767321067.745880:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321067.745884:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321067.745904:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2300 x1853163985585664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321067.746525:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985585664:0@lo:38:kworker.0 00000100:00100000:1.0:1767321067.746711:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321067.746819:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985585664 00000100:00100000:2.0:1767321067.746827:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985585664 00000100:00080000:3.0:1767321067.747539:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321067.747544:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321067.747548:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321067.747553:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6310 00000100:00080000:3.0:1767321067.747557:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767321067.747736:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6402 00000100:00100000:2.0:1767321067.747744:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985585664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321067.747765:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321067.747817:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985585664:12345-0@lo:38:kworker.0 Request processed in 73us (1106us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321067.747825:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6402 00000100:00080000:3.0:1767321067.747858:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321067.748720:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2d80 x1853163985585792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321067.748763:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321067.748767:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321067.748770:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321067.748773:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6310 00000100:00080000:3.0:1767321067.748776:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321067.748780:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321067.748793:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc1f80 x1853163985586048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321067.748808:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321067.748811:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321067.748814:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321067.748816:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6310 00000100:00080000:3.0:1767321067.748819:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321067.748821:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321067.748830:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3800 x1853163985586304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321067.749220:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985586176:0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.749274:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.749306:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321067.749312:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985585920:0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.749330:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.749355:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985586432:0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.749379:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.749395:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321067.749400:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985586560:0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.749426:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.749450:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985585792:0@lo:38:kworker.0 00000100:00100000:0.0:1767321067.749474:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.749490:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985586048:0@lo:38:kworker.0 00000100:00100000:0.0:1767321067.749513:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.749530:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985586304:0@lo:38:kworker.0 00000100:00100000:0.0:1767321067.749550:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321067.749586:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321067.749591:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321067.749595:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985585664:0@lo:38:kworker.0 00000100:00100000:0.0:1767321067.749640:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985585792 00000100:00100000:0.0:1767321067.749647:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985585792 00000100:00100000:0.0:1767321067.749679:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2091 00000100:00100000:0.0:1767321067.749685:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985585792:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321067.749702:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321067.749720:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586176 00000100:00100000:2.0:1767321067.749726:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586176 00000100:00100000:0.0:1767321067.749741:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985585792:12345-0@lo:38:kworker.0 Request processed in 53us (264us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321067.749749:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2091 00000100:00100000:2.0:1767321067.749773:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985585920 00000100:00100000:0.0:1767321067.749777:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586048 00000100:00100000:2.0:1767321067.749778:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985585920 00000100:00100000:0.0:1767321067.749784:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586048 00000100:00100000:2.0:1767321067.749796:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586304 00000100:00100000:2.0:1767321067.749801:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586304 00000100:00100000:0.0:1767321067.749805:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5598 00000100:00100000:0.0:1767321067.749816:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985586176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321067.749820:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5599 00000100:00100000:2.0:1767321067.749832:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985585920:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.749870:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985586176:12345-0@lo:400:kworker.0 Request processed in 57us (595us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321067.749877:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5598 00000100:00100000:0.0:1767321067.749892:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6403 00000100:00100000:0.0:1767321067.749896:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce187fe8c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985586048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321067.749906:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321067.749928:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce187fe8c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985586048:12345-0@lo:38:kworker.0 Request processed in 31us (415us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321067.749934:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6403 00000100:00100000:0.0:1767321067.749944:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6404 00000100:00100000:0.0:1767321067.749948:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985586304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321067.749957:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321067.749975:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985586304:12345-0@lo:38:kworker.0 Request processed in 26us (425us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321067.749981:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6404 00000100:00080000:0.0:1767321067.749999:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321067.750020:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985586176:0@lo:400:kworker.0 00000100:00080000:0.0:1767321067.750038:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321067.750042:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321067.750045:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985585792:0@lo:38:kworker.0 00000100:00080000:0.0:1767321067.750052:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321067.750055:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321067.750058:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985586048:0@lo:38:kworker.0 00000100:00080000:0.0:1767321067.750065:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321067.750068:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321067.750071:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985586304:0@lo:38:kworker.0 00000100:00100000:1.0:1767321067.750409:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586432 00000100:00100000:1.0:1767321067.750415:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586432 00000100:00100000:2.0:1767321067.750786:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985585920:12345-0@lo:400:kworker.0 Request processed in 954us (1452us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321067.750794:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5599 00000100:00100000:0.0:1767321067.751593:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985585920:0@lo:400:kworker.0 00000100:00100000:3.0:1767321067.751607:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586560 00000100:00100000:3.0:1767321067.751614:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586560 00000100:00100000:0.0:1767321067.751682:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3625 00000100:00100000:0.0:1767321067.751692:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985586560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.751757:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985586560:12345-0@lo:400:kworker.0 Request processed in 68us (2330us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321067.751764:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3625 00000100:00100000:3.0:1767321067.751910:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985586560:0@lo:400:kworker.0 00000100:00100000:1.0:1767321067.752084:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3626 00000100:00100000:1.0:1767321067.752093:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985586432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321067.752143:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985586432:12345-0@lo:400:kworker.0 Request processed in 52us (2763us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321067.752149:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3626 00000100:00100000:3.0:1767321067.752230:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985586432:0@lo:400:kworker.0 00010000:00010000:2.0:1767321068.379812:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321069.403525:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321070.427859:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321071.323584:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321071.323610:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985586688:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321071.323682:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321071.325891:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586688 00000100:00100000:2.0:1767321071.325904:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586688 00000100:00100000:2.0:1767321071.325973:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2842 00000100:00100000:2.0:1767321071.325992:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985586688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321071.326024:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321071.326114:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985586688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 132us (2431us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321071.326120:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2842 00000100:00100000:3.0:1767321071.326202:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985586688:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321071.328354:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321071.328424:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321071.328441:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985586816:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321071.328503:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321071.328602:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586816 00000100:00100000:0.0:1767321071.328610:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586816 00000100:00100000:0.0:1767321071.328670:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2843 00000100:00100000:0.0:1767321071.328680:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985586816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321071.328706:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321071.328756:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985586816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (252us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321071.328763:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2843 00000100:00080000:2.0:1767321071.328824:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321071.328835:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985586816:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321071.453275:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321072.475835:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321072.809767:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321072.810076:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920652800 00000100:00100000:2.0:1767321072.810086:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920652800 00000100:00100000:2.0:1767321072.810286:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5600 00000100:00100000:2.0:1767321072.810300:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920652800:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321072.810386:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920652800:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (629us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321072.810394:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5600 00000100:00080000:0.0:1767321072.810719:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321072.811590:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321072.811818:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920652672 00000100:00100000:3.0:1767321072.811932:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920652672 00000100:00100000:3.0:1767321072.811973:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6405 00000100:00100000:3.0:1767321072.811981:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920652672:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321072.812002:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321072.816734:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920652672:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 4737us (5133us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321072.816751:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6405 00000100:00080000:1.0:1767321072.859641:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321072.859652:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321072.859655:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321072.859659:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321072.859663:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321072.859671:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6316 00000100:00080000:1.0:1767321072.859677:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321072.859684:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321072.859727:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c6680 x1853163985586944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321072.859795:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985586944:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.859857:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321072.859923:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321072.859926:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321072.859929:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321072.859932:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6316 00000100:00080000:1.0:1767321072.859933:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321072.859936:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321072.859949:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c7800 x1853163985587072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321072.860028:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985586944 00000100:00100000:3.0:1767321072.860039:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985586944 00000100:00100000:3.0:1767321072.860082:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6406 00000100:00100000:3.0:1767321072.860090:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985586944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321072.860110:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321072.861335:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985587072:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.861669:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321072.861809:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321072.861814:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321072.861816:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321072.861821:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6316 00000100:00080000:1.0:1767321072.861824:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321072.861829:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321072.861847:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4700 x1853163985587328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321072.861864:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321072.861867:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321072.861869:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321072.861873:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6316 00000100:00080000:1.0:1767321072.861875:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321072.861878:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321072.861888:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5880 x1853163985587584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321072.861936:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985587200:0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.861967:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321072.862007:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321072.862013:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985587456:0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.862035:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321072.862062:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985587712:0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.862081:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321072.862095:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321072.862098:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985587840:0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.862109:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321072.862128:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587200 00000100:00100000:1.0:1767321072.862133:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587200 00000100:00100000:1.0:1767321072.863023:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587456 00000100:00100000:1.0:1767321072.863030:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587456 00000100:00100000:1.0:1767321072.863058:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5601 00000100:00100000:1.0:1767321072.863066:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985587200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.863123:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985587200:12345-0@lo:400:kworker.0 Request processed in 56us (1154us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321072.863129:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5601 00000100:00100000:1.0:1767321072.863146:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5602 00000100:00100000:3.0:1767321072.863277:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985586944:12345-0@lo:38:kworker.0 Request processed in 3185us (3417us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321072.863285:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6406 00000100:00100000:3.0:1767321072.863345:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587712 00000100:00100000:3.0:1767321072.863350:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587712 00000100:00100000:3.0:1767321072.863376:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587840 00000100:00100000:3.0:1767321072.863379:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587840 00000100:00100000:3.0:1767321072.863413:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3627 00000100:00100000:3.0:1767321072.863420:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985587712:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321072.863455:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985587712:12345-0@lo:400:kworker.0 Request processed in 36us (1372us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321072.863460:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3627 00000100:00100000:3.0:1767321072.863485:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3628 00000100:00100000:3.0:1767321072.863490:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985587840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321072.863517:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985587840:12345-0@lo:400:kworker.0 Request processed in 27us (1406us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321072.863522:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3628 00000100:00080000:3.0:1767321072.863551:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321072.863605:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985587200:0@lo:400:kworker.0 00000100:00100000:3.0:1767321072.863657:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985587840:0@lo:400:kworker.0 00000100:00100000:3.0:1767321072.863667:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985587712:0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.863932:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985587456:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.863978:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985587456:12345-0@lo:400:kworker.0 Request processed in 815us (1942us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321072.863984:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5602 00000100:00100000:3.0:1767321072.864125:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985587456:0@lo:400:kworker.0 00000100:00100000:2.0:1767321072.864306:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587072 00000100:00100000:2.0:1767321072.864312:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587072 00000100:00100000:2.0:1767321072.864336:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2092 00000100:00100000:2.0:1767321072.864342:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985587072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321072.864360:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321072.864581:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985587072:12345-0@lo:38:kworker.0 Request processed in 238us (2912us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321072.864588:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2092 00000100:00080000:0.0:1767321072.865860:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.865869:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.865873:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985586944:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.865906:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985587328:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.865955:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321072.866024:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985587584:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.866046:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321072.866120:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.866125:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.866128:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985587072:0@lo:38:kworker.0 00000100:00100000:1.0:1767321072.866412:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587328 00000100:00100000:1.0:1767321072.866418:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587328 00000100:00100000:1.0:1767321072.866455:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587584 00000100:00100000:1.0:1767321072.866458:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587584 00000100:00100000:1.0:1767321072.866669:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6407 00000100:00100000:1.0:1767321072.866676:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985587328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321072.866694:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321072.866742:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985587328:12345-0@lo:38:kworker.0 Request processed in 65us (787us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321072.866748:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6407 00000100:00100000:1.0:1767321072.866762:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6408 00000100:00100000:1.0:1767321072.866765:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985587584:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321072.866773:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321072.866793:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985587584:12345-0@lo:38:kworker.0 Request processed in 26us (746us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321072.866798:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6408 00000100:00080000:0.0:1767321072.867828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.867836:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.867840:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985587328:0@lo:38:kworker.0 00000100:00080000:0.0:1767321072.867850:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.867853:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.867855:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985587584:0@lo:38:kworker.0 00010000:00010000:2.0:1767321073.499637:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321074.524753:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321075.547447:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321076.443692:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985587968:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321076.443766:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321076.443868:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985587968 00000100:00100000:0.0:1767321076.443877:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985587968 00000100:00100000:0.0:1767321076.443948:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2844 00000100:00100000:0.0:1767321076.443957:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985587968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321076.443987:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321076.444028:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985587968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (263us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321076.444035:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2844 00000100:00100000:3.0:1767321076.444055:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985588096:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321076.444091:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321076.444125:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321076.444190:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588096 00000100:00100000:0.0:1767321076.444195:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588096 00000100:00100000:0.0:1767321076.444235:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2845 00000100:00100000:0.0:1767321076.444242:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985588096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321076.444257:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321076.444286:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985588096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (195us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321076.444292:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2845 00000100:00080000:1.0:1767321076.444339:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321076.444720:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985587968:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321076.446046:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985588096:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321076.573498:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321077.595864:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321077.930048:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321077.930108:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321077.930337:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920652928 00000100:00100000:3.0:1767321077.930348:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920652928 00000100:00100000:3.0:1767321077.930398:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920653056 00000100:00100000:3.0:1767321077.930402:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920653056 00000100:00100000:3.0:1767321077.930472:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5603 00000100:00100000:3.0:1767321077.930483:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920653056:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:0.0:1767321077.930495:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321077.930517:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6409 00000100:00100000:0.0:1767321077.930524:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920652928:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321077.930547:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321077.930569:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920653056:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (461us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321077.930579:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5603 00000100:00100000:0.0:1767321077.930583:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920652928:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (542us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321077.930593:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6409 00000100:00080000:2.0:1767321077.983126:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321077.983137:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321077.983140:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321077.983147:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321077.983152:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321077.983159:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6321 00000100:00080000:2.0:1767321077.983178:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321077.983185:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321077.983229:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0380 x1853163985588224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321077.983627:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985588224:0@lo:38:kworker.0 00000100:00100000:0.0:1767321077.983697:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321077.983948:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588224 00000100:00100000:1.0:1767321077.983956:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588224 00000100:00100000:1.0:1767321077.984007:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6410 00000100:00100000:1.0:1767321077.984014:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985588224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321077.984038:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321077.984218:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321077.984223:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321077.984226:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321077.984230:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6321 00000100:00080000:2.0:1767321077.984232:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321077.984237:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321077.984268:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2d80 x1853163985588352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321077.987839:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985588352:0@lo:38:kworker.0 00000100:00100000:0.0:1767321077.987914:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321077.988045:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588352 00000100:00100000:3.0:1767321077.988054:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588352 00000100:00100000:3.0:1767321077.988097:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2093 00000100:00100000:3.0:1767321077.988104:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985588352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321077.988125:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321077.988186:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985588352:12345-0@lo:38:kworker.0 Request processed in 80us (270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321077.988193:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2093 00000100:00100000:1.0:1767321077.988562:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985588224:12345-0@lo:38:kworker.0 Request processed in 4545us (4863us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321077.988571:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6410 00000100:00080000:0.0:1767321077.991486:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321077.991500:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321077.991505:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985588224:0@lo:38:kworker.0 00000100:00080000:0.0:1767321077.991546:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321077.991550:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321077.991553:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985588352:0@lo:38:kworker.0 00000100:00100000:3.0:1767321077.992086:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985588480:0@lo:400:kworker.0 00000100:00100000:3.0:1767321077.992148:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321077.992303:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588480 00000100:00100000:1.0:1767321077.992318:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588480 00000100:00100000:1.0:1767321077.992384:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5604 00000100:00100000:1.0:1767321077.992394:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985588480:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321077.992559:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321077.992635:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321077.992641:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321077.992644:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321077.992648:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6321 00000100:00080000:2.0:1767321077.992652:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321077.992656:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321077.992679:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1880 x1853163985588608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321077.992754:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985588736:0@lo:400:kworker.0 00000100:00100000:0.0:1767321077.992783:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321077.992814:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588736 00000100:00100000:0.0:1767321077.992818:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588736 00000100:00100000:0.0:1767321077.992848:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5605 00000100:00100000:0.0:1767321077.992854:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985588736:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321077.992885:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985588736:12345-0@lo:400:kworker.0 Request processed in 31us (102us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321077.992893:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5605 00000100:00080000:0.0:1767321077.992912:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321077.992934:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985588736:0@lo:400:kworker.0 00000100:00100000:0.0:1767321077.992956:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985588608:0@lo:38:kworker.0 00000100:00100000:0.0:1767321077.992980:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321077.993006:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588608 00000100:00100000:0.0:1767321077.993009:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588608 00000100:00100000:0.0:1767321077.993028:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6411 00000100:00100000:0.0:1767321077.993032:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985588608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321077.993050:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321077.993080:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985588608:12345-0@lo:38:kworker.0 Request processed in 46us (99us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321077.993086:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6411 00000100:00080000:0.0:1767321077.993109:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321077.993113:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321077.993116:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985588608:0@lo:38:kworker.0 00000100:00100000:1.0:1767321077.994552:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985588480:12345-0@lo:400:kworker.0 Request processed in 2154us (2398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321077.994564:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5604 00000100:00080000:2.0:1767321077.998034:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321077.998048:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321077.998053:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321077.998061:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6321 00000100:00080000:2.0:1767321077.998066:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321077.998073:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321077.998115:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985588864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321077.998208:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985588864:0@lo:38:kworker.0 00000100:00100000:0.0:1767321077.998275:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321077.998378:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588864 00000100:00100000:1.0:1767321077.998386:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588864 00000100:00100000:1.0:1767321077.998435:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6412 00000100:00100000:1.0:1767321077.998454:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985588864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321077.998474:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321077.998587:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985589120:0@lo:400:kworker.0 00000100:00100000:2.0:1767321077.998619:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321077.998643:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321077.998649:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985588992:0@lo:400:kworker.0 00000100:00100000:2.0:1767321077.998669:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321077.998703:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985589120 00000100:00100000:2.0:1767321077.999868:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985589120 00000100:00100000:2.0:1767321077.999927:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985588992 00000100:00100000:2.0:1767321077.999931:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985588992 00000100:00100000:2.0:1767321077.999956:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3629 00000100:00100000:2.0:1767321077.999965:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985589120:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321078.000040:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985589120:12345-0@lo:400:kworker.0 Request processed in 76us (1419us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321078.000047:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3629 00000100:00100000:2.0:1767321078.000064:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3630 00000100:00100000:2.0:1767321078.000070:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985588992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321078.000098:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985588992:12345-0@lo:400:kworker.0 Request processed in 28us (1428us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321078.000103:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3630 00000100:00100000:2.0:1767321078.000156:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985588992:0@lo:400:kworker.0 00000100:00100000:2.0:1767321078.000188:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985589120:0@lo:400:kworker.0 00000100:00100000:3.0:1767321078.000342:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985588480:0@lo:400:kworker.0 00000100:00100000:1.0:1767321078.003674:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985588864:12345-0@lo:38:kworker.0 Request processed in 5210us (5389us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321078.003688:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6412 00000100:00080000:0.0:1767321078.003935:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321078.003941:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321078.003945:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985588864:0@lo:38:kworker.0 00010000:00010000:2.0:1767321078.619479:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321079.643833:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321080.667347:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321081.563616:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321081.563639:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985589248:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321081.563710:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321081.563834:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985589248 00000100:00100000:0.0:1767321081.563842:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985589248 00000100:00100000:0.0:1767321081.563908:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2846 00000100:00100000:0.0:1767321081.563917:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985589248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321081.563946:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321081.563989:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985589248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321081.563996:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2846 00000100:00080000:2.0:1767321081.564067:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321081.564105:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985589248:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321081.564250:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985589376:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321081.564275:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321081.564350:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985589376 00000100:00100000:0.0:1767321081.564355:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985589376 00000100:00100000:0.0:1767321081.564407:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2847 00000100:00100000:0.0:1767321081.564413:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985589376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321081.564440:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321081.564473:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985589376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (195us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321081.564478:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2847 00000100:00080000:2.0:1767321081.564534:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321081.564557:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985589376:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321081.691954:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321082.717946:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321083.051254:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321083.051564:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321083.051838:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920653312 00000100:00100000:3.0:1767321083.051850:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920653312 00000100:00100000:3.0:1767321083.051924:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920653184 00000100:00100000:3.0:1767321083.051928:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920653184 00000100:00100000:3.0:1767321083.051948:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5606 00000100:00100000:3.0:1767321083.051957:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920653312:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321083.052031:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920653312:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (799us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321083.052038:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5606 00000100:00100000:3.0:1767321083.052056:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6413 00000100:00100000:3.0:1767321083.052060:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920653184:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321083.052079:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321083.052100:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920653184:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 38us (538us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321083.052105:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6413 00000100:00080000:0.0:1767321083.053692:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321083.099502:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321083.099511:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321083.099514:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321083.099518:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321083.099522:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321083.099531:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6326 00000100:00080000:2.0:1767321083.099535:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321083.099542:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321083.099584:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0000 x1853163985589504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321083.099624:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321083.099627:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321083.099629:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321083.099632:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6326 00000100:00080000:2.0:1767321083.099635:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321083.099638:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321083.099652:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3100 x1853163985589632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321083.099696:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321083.099700:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321083.099702:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321083.099705:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6326 00000100:00080000:2.0:1767321083.099708:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321083.099711:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321083.099722:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2300 x1853163985589888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321083.099736:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321083.099739:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321083.099741:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321083.099744:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6326 00000100:00080000:2.0:1767321083.099746:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321083.099749:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321083.099763:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0380 x1853163985590144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321083.099828:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985590272:0@lo:400:kworker.0 00000100:00100000:2.0:1767321083.099897:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321083.099958:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321083.099966:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985590400:0@lo:400:kworker.0 00000100:00100000:2.0:1767321083.099988:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321083.100020:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590272 00000100:00100000:2.0:1767321083.100026:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590272 00000100:00100000:2.0:1767321083.100083:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590400 00000100:00100000:2.0:1767321083.100086:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590400 00000100:00100000:2.0:1767321083.100108:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3631 00000100:00100000:2.0:1767321083.100117:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985590272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100379:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985589760:0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100414:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321083.100440:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321083.100446:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985590016:0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100465:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321083.100492:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985589760 00000100:00100000:3.0:1767321083.100496:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985589760 00000100:00100000:3.0:1767321083.100523:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590016 00000100:00100000:3.0:1767321083.100526:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590016 00000100:00100000:3.0:1767321083.100544:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5607 00000100:00100000:3.0:1767321083.100551:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985589760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100587:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985589760:12345-0@lo:400:kworker.0 Request processed in 36us (172us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321083.100593:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5607 00000100:00100000:3.0:1767321083.100609:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5608 00000100:00100000:3.0:1767321083.100615:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985590016:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100640:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985590016:12345-0@lo:400:kworker.0 Request processed in 25us (174us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321083.100645:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5608 00000100:00100000:3.0:1767321083.100685:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985590016:0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100696:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985589760:0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100742:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3632 00000100:00100000:3.0:1767321083.100750:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985590400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321083.100786:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985590400:12345-0@lo:400:kworker.0 Request processed in 38us (797us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321083.100792:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3632 00000100:00100000:3.0:1767321083.100821:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985590400:0@lo:400:kworker.0 00000100:00100000:0.0:1767321083.101040:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985589504:0@lo:38:kworker.0 00000100:00100000:0.0:1767321083.101072:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321083.101103:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985589632:0@lo:38:kworker.0 00000100:00100000:0.0:1767321083.101125:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321083.101161:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985589888:0@lo:38:kworker.0 00000100:00100000:0.0:1767321083.101205:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321083.101222:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985590144:0@lo:38:kworker.0 00000100:00100000:0.0:1767321083.101242:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321083.101272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321083.101276:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321083.101279:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321083.101291:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985589632 00000100:00100000:0.0:1767321083.101294:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985589632 00000100:00100000:0.0:1767321083.101318:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2094 00000100:00100000:0.0:1767321083.101322:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985589632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321083.101340:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321083.101406:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985589632:12345-0@lo:38:kworker.0 Request processed in 81us (279us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321083.101415:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2094 00000100:00100000:0.0:1767321083.101437:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985589504 00000100:00100000:0.0:1767321083.101440:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985589504 00000100:00100000:0.0:1767321083.101459:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985589888 00000100:00100000:0.0:1767321083.101461:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985589888 00000100:00100000:0.0:1767321083.101471:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590144 00000100:00100000:0.0:1767321083.101474:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590144 00000100:00100000:0.0:1767321083.101483:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6414 00000100:00100000:0.0:1767321083.101486:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985589504:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321083.101497:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321083.101522:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985589504:12345-0@lo:38:kworker.0 Request processed in 34us (449us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321083.101527:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6414 00000100:00100000:0.0:1767321083.101538:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6415 00000100:00100000:0.0:1767321083.101542:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985589888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321083.101550:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321083.101570:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985589888:12345-0@lo:38:kworker.0 Request processed in 27us (365us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321083.101576:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6415 00000100:00100000:0.0:1767321083.101587:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6416 00000100:00100000:0.0:1767321083.101591:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985590144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321083.101599:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321083.101619:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985590144:12345-0@lo:38:kworker.0 Request processed in 26us (376us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321083.101624:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6416 00000100:00080000:0.0:1767321083.101670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321083.101677:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321083.101680:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985589504:0@lo:38:kworker.0 00000100:00080000:0.0:1767321083.101689:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321083.101692:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321083.101695:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985589632:0@lo:38:kworker.0 00000100:00080000:0.0:1767321083.101701:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321083.101704:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321083.101706:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985589888:0@lo:38:kworker.0 00000100:00080000:0.0:1767321083.101713:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321083.101716:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321083.101719:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985590144:0@lo:38:kworker.0 00000100:00100000:2.0:1767321083.102752:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985590272:12345-0@lo:400:kworker.0 Request processed in 2631us (2849us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321083.102762:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3631 00000100:00100000:3.0:1767321083.103228:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985590272:0@lo:400:kworker.0 00010000:00010000:2.0:1767321083.739406:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321084.763381:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321085.787422:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321086.684769:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321086.684793:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985590528:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321086.684859:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321086.684965:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590528 00000100:00100000:1.0:1767321086.684972:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590528 00000100:00100000:1.0:1767321086.685031:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2848 00000100:00100000:1.0:1767321086.685039:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985590528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321086.685066:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321086.685106:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985590528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (246us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321086.685113:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2848 00000100:00080000:2.0:1767321086.685363:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321086.685376:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985590656:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321086.685400:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321086.685461:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985590528:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321086.685522:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590656 00000100:00100000:1.0:1767321086.685526:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590656 00000100:00100000:1.0:1767321086.685560:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2849 00000100:00100000:1.0:1767321086.685565:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985590656:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321086.685580:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321086.685607:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985590656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (206us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321086.685613:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2849 00000100:00080000:2.0:1767321086.685999:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321086.686022:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985590656:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321086.811675:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321087.835994:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321088.170932:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321088.171364:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321088.172691:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920653568 00000100:00100000:1.0:1767321088.172702:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920653568 00000100:00100000:1.0:1767321088.172759:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920653440 00000100:00100000:1.0:1767321088.172763:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920653440 00000100:00100000:1.0:1767321088.172781:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5609 00000100:00100000:1.0:1767321088.172790:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920653568:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321088.172878:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920653568:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (1957us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321088.172886:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5609 00000100:00100000:1.0:1767321088.172904:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6417 00000100:00100000:1.0:1767321088.172908:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920653440:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321088.172928:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321088.172966:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920653440:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (1602us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321088.172972:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6417 00000100:00080000:2.0:1767321088.175212:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321088.219353:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321088.219363:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321088.219366:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321088.219370:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321088.219374:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321088.219384:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6331 00000100:00080000:2.0:1767321088.219389:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321088.219395:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321088.219442:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985590784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321088.219460:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321088.219463:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321088.219464:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321088.219467:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6331 00000100:00080000:2.0:1767321088.219469:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321088.219472:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321088.219486:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0700 x1853163985590912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321088.219573:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767321088.219593:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321088.219597:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321088.219599:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321088.219603:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6331 00000100:00080000:2.0:1767321088.219606:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321088.219609:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321088.219622:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3800 x1853163985591168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321088.219694:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321088.219705:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985591296:0@lo:400:kworker.0 00000100:00100000:3.0:1767321088.219782:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321088.219861:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591296 00000100:00100000:3.0:1767321088.219868:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591296 00000100:00100000:3.0:1767321088.219919:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5610 00000100:00100000:3.0:1767321088.219928:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985591296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321088.219978:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985591296:12345-0@lo:400:kworker.0 Request processed in 51us (195us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321088.219986:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5610 00000100:00080000:3.0:1767321088.220008:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321088.220041:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985591296:0@lo:400:kworker.0 00000100:00100000:0.0:1767321088.220280:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985590784:0@lo:38:kworker.0 00000100:00100000:0.0:1767321088.220310:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321088.220343:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985590912:0@lo:38:kworker.0 00000100:00100000:0.0:1767321088.220366:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321088.220389:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985591168:0@lo:38:kworker.0 00000100:00100000:0.0:1767321088.220410:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321088.220439:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590912 00000100:00100000:0.0:1767321088.220443:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590912 00000100:00100000:0.0:1767321088.220465:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2095 00000100:00100000:0.0:1767321088.220470:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985590912:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321088.220490:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321088.220521:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985590912:12345-0@lo:38:kworker.0 Request processed in 50us (153us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321088.220527:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2095 00000100:00080000:0.0:1767321088.220553:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321088.220558:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321088.220561:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985590912:0@lo:38:kworker.0 00000100:00100000:0.0:1767321088.220579:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985590784 00000100:00100000:0.0:1767321088.220582:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985590784 00000100:00100000:0.0:1767321088.220599:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591168 00000100:00100000:0.0:1767321088.220601:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591168 00000100:00100000:0.0:1767321088.220617:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6418 00000100:00100000:0.0:1767321088.220621:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985590784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321088.220630:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321088.220657:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985590784:12345-0@lo:38:kworker.0 Request processed in 34us (347us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321088.220663:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6418 00000100:00100000:0.0:1767321088.220674:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6419 00000100:00100000:0.0:1767321088.220678:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985591168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321088.220687:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321088.220708:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985591168:12345-0@lo:38:kworker.0 Request processed in 29us (298us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321088.220713:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6419 00000100:00080000:0.0:1767321088.220735:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321088.220739:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321088.220743:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985590784:0@lo:38:kworker.0 00000100:00080000:0.0:1767321088.220750:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321088.220753:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321088.220756:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985591168:0@lo:38:kworker.0 00000100:00100000:1.0:1767321088.221655:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985591040:0@lo:400:kworker.0 00000100:00100000:1.0:1767321088.221697:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321088.221844:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321088.221848:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321088.221851:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321088.221855:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6331 00000100:00080000:2.0:1767321088.221859:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321088.221863:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321088.221887:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff2a00 x1853163985591424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321088.221935:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985591552:0@lo:400:kworker.0 00000100:00100000:0.0:1767321088.222194:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985591680:0@lo:400:kworker.0 00000100:00100000:0.0:1767321088.222232:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321088.222272:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985591424:0@lo:38:kworker.0 00000100:00100000:0.0:1767321088.222291:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321088.222344:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591680 00000100:00100000:1.0:1767321088.222351:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591680 00000100:00100000:1.0:1767321088.222399:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3633 00000100:00100000:1.0:1767321088.222407:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985591680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321088.222453:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985591680:12345-0@lo:400:kworker.0 Request processed in 47us (219us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321088.222463:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3633 00000100:00100000:1.0:1767321088.222499:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985591680:0@lo:400:kworker.0 00000100:00080000:1.0:1767321088.222535:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321088.222678:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321088.222738:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591552 00000100:00100000:1.0:1767321088.222742:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591552 00000100:00100000:1.0:1767321088.222778:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3634 00000100:00100000:1.0:1767321088.222784:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985591552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321088.222827:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985591552:12345-0@lo:400:kworker.0 Request processed in 43us (148us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321088.222833:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3634 00000100:00080000:2.0:1767321088.222885:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321088.222904:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985591552:0@lo:400:kworker.0 00000100:00100000:3.0:1767321088.223320:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591040 00000100:00100000:3.0:1767321088.223326:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591040 00000100:00100000:3.0:1767321088.223373:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591424 00000100:00100000:3.0:1767321088.223377:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591424 00000100:00100000:3.0:1767321088.223396:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5611 00000100:00100000:3.0:1767321088.223403:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985591040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321088.223445:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985591040:12345-0@lo:400:kworker.0 Request processed in 43us (1748us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321088.223452:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5611 00000100:00100000:3.0:1767321088.223466:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6420 00000100:00100000:3.0:1767321088.223470:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985591424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321088.223487:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321088.223519:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985591424:12345-0@lo:38:kworker.0 Request processed in 47us (1226us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321088.223524:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6420 00000100:00080000:3.0:1767321088.223554:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321088.223559:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321088.223562:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985591424:0@lo:38:kworker.0 00000100:00080000:2.0:1767321088.223631:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321088.223703:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985591040:0@lo:400:kworker.0 00010000:00010000:1.0:1767321088.859513:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321089.883809:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321090.907389:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321091.803557:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985591808:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321091.803627:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321091.803675:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591808 00000100:00100000:0.0:1767321091.803683:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591808 00000100:00100000:3.0:1767321091.803696:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985591936:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321091.803724:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321091.803760:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985591936 00000100:00100000:0.0:1767321091.803764:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985591936 00000100:00100000:0.0:1767321091.803814:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2850 00000100:00100000:0.0:1767321091.803824:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985591808:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321091.803828:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2851 00000100:00100000:1.0:1767321091.803842:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985591936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321091.803868:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:1.0:1767321091.803868:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767321091.803950:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321091.803954:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321091.804503:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985591936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 663us (776us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321091.804512:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2851 00000100:00100000:0.0:1767321091.804522:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985591808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 697us (892us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321091.804531:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2850 00000100:00100000:3.0:1767321091.804633:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985591808:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321091.804651:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985591936:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321091.933642:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321092.266817:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767321092.266853:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920653696 offset 0 length 520: 4 00000100:00100000:1.0:1767321092.266878:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321092.266901:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920653952 00000100:00100000:0.0:1767321092.266915:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920653952 00000100:00100000:2.0:1767321092.266946:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920653824 00000100:00100000:2.0:1767321092.266952:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920653824 00000100:00100000:2.0:1767321092.267014:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5612 00000100:00100000:0.0:1767321092.267024:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6421 00000100:00100000:0.0:1767321092.267036:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920653824:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:2.0:1767321092.267132:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920653952:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321092.267260:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920653952:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 235us (452us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321092.267268:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5612 00000100:00080000:1.0:1767321092.267726:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:0.0:1767321092.267825:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321092.267875:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920653824:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 841us (996us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321092.267883:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6421 00000100:00100000:3.0:1767321092.315320:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321067/real 1767321067] req@ffff8ce1b2dc0a80 x1853163985585536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321092 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321092.315354:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b2dc0a80 x1853163985585536/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321092 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321092.315374:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321092.315397:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321092.315401:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985585536:0@lo:250:kworker.0 00000100:00080000:2.0:1767321092.315544:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321092.315549:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767321092.315552:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321092.315557:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6316 00000100:00080000:2.0:1767321092.315561:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321092.315567:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321092.315598:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1c00 x1853163985592064/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321092.315620:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321092.315624:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321092.315626:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321092.315630:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6336 00000100:00080000:2.0:1767321092.315632:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321092.315635:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321092.315651:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1180 x1853163985592192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321092.315658:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321092.315661:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321092.315663:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321092.315666:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6336 00000100:00080000:2.0:1767321092.315668:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321092.315671:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321092.315905:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985592064:0@lo:250:kworker.0 00000400:00000100:3.0:1767321092.315953:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985592064 offset 0 length 520: 4 00000100:00100000:3.0:1767321092.316217:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985592192:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.316265:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321092.316333:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985592192 00000100:00100000:0.0:1767321092.316341:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985592192 00000100:00100000:2.0:1767321092.316355:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3b80 x1853163985592320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321092.316390:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6422 00000100:00100000:0.0:1767321092.316398:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985592192:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767321092.316400:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321092.316404:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321092.316406:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321092.316410:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6336 00000100:00080000:2.0:1767321092.316413:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321092.316416:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00010000:00020000:0.0:1767321092.316418:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321092.316420:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321092.316431:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1f80 x1853163985592576/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321092.316434:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985592448:0@lo:400:kworker.0 00000100:00080000:2.0:1767321092.316469:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767321092.316472:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985592192:12345-0@lo:38:kworker.0 Request processed in 73us (207us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767321092.316472:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321092.316474:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321092.316477:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6336 00000100:00100000:0.0:1767321092.316479:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6422 00000100:00080000:2.0:1767321092.316479:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321092.316482:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321092.316495:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985592832/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321092.316517:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985592704:0@lo:400:kworker.0 00000100:00100000:2.0:1767321092.316519:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985592320:0@lo:38:kworker.0 00000100:00100000:0.0:1767321092.318558:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321092.318661:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321092.318841:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985592576:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.318870:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985592320 00000100:00100000:3.0:1767321092.318880:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985592320 00000100:00100000:3.0:1767321092.318925:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2096 00000100:00100000:3.0:1767321092.318934:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985592320:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321092.318955:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321092.319091:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985592320:12345-0@lo:38:kworker.0 Request processed in 156us (428us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321092.319098:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2096 00000100:00100000:3.0:1767321092.319143:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321092.319474:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321092.322924:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985592704 00000100:00100000:1.0:1767321092.322940:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985592704 00000100:00100000:2.0:1767321092.322955:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985592832:0@lo:38:kworker.0 00000100:00100000:1.0:1767321092.323015:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985592448 00000100:00100000:1.0:1767321092.323019:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985592448 00000100:00100000:2.0:1767321092.323024:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321092.323048:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985592576 00000100:00100000:1.0:1767321092.323053:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985592576 00000100:00100000:1.0:1767321092.323064:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985592832 00000100:00100000:1.0:1767321092.323066:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985592832 00000100:00080000:2.0:1767321092.323074:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321092.323080:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321092.323085:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5613 00000100:00100000:2.0:1767321092.323089:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985592192:0@lo:38:kworker.0 00000100:00100000:1.0:1767321092.323097:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985592704:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321092.323128:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321092.323134:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321092.323137:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985592320:0@lo:38:kworker.0 00000100:00100000:1.0:1767321092.323198:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985592704:12345-0@lo:400:kworker.0 Request processed in 68us (4612us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321092.323207:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5613 00000100:00100000:2.0:1767321092.323221:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5614 00000100:00100000:1.0:1767321092.323230:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6423 00000100:00100000:2.0:1767321092.323235:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985592448:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321092.323241:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985592576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321092.323262:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321092.323384:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985592576:12345-0@lo:38:kworker.0 Request processed in 144us (3909us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321092.323391:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6423 00000100:00100000:1.0:1767321092.323403:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6424 00000100:00100000:1.0:1767321092.323406:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985592832:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321092.323415:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321092.323538:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985592832:12345-0@lo:38:kworker.0 Request processed in 26us (413us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321092.323544:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6424 00000100:00100000:1.0:1767321092.323578:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985592704:0@lo:400:kworker.0 00000100:00080000:1.0:1767321092.323613:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321092.323825:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321092.323830:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321092.323833:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985592576:0@lo:38:kworker.0 00000100:00080000:0.0:1767321092.323840:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321092.323843:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321092.323845:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985592832:0@lo:38:kworker.0 00000100:00100000:2.0:1767321092.324412:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985592448:12345-0@lo:400:kworker.0 Request processed in 1182us (5266us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321092.324422:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5614 00000100:00100000:0.0:1767321092.324660:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985592448:0@lo:400:kworker.0 00010000:00010000:2.0:1767321092.957392:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321093.979470:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321095.003489:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321096.028134:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321096.923556:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985592960:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321096.923571:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321096.923590:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985593088:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321096.923818:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321096.924248:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593088 00000100:00100000:2.0:1767321096.924258:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593088 00000100:00100000:2.0:1767321096.924376:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2852 00000100:00100000:2.0:1767321096.924386:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985593088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321096.924414:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321096.924465:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985593088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (648us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321096.924473:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2852 00000100:00100000:2.0:1767321096.924518:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985593088:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321096.924742:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321096.929838:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321096.940195:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985592960 00000100:00100000:3.0:1767321096.940212:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985592960 00000100:00080000:1.0:1767321096.940509:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321096.940519:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2853 00000100:00100000:3.0:1767321096.940531:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985592960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321096.940567:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321096.940677:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985592960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 149us (10849us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321096.940686:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2853 00000100:00100000:0.0:1767321096.940781:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985592960:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321097.052058:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321097.386401:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321097.386464:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321097.386674:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654080 00000100:00100000:0.0:1767321097.386685:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654080 00000100:00100000:0.0:1767321097.386744:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654208 00000100:00100000:0.0:1767321097.386747:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654208 00000100:00100000:0.0:1767321097.386794:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5615 00000100:00100000:0.0:1767321097.386805:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920654208:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321097.386860:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6425 00000100:00100000:3.0:1767321097.386867:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920654080:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:0.0:1767321097.386885:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920654208:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 81us (419us total) trans 0 rc 0/0 00010000:00020000:3.0:1767321097.386891:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321097.386908:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5615 00000100:00100000:3.0:1767321097.386924:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920654080:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 55us (533us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321097.386933:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6425 00000100:00080000:0.0:1767321097.386942:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321097.436922:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321097.436931:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321097.436934:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321097.436938:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321097.436942:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321097.436949:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6340 00000100:00080000:0.0:1767321097.436953:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321097.436960:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321097.436997:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bebb80 x1853163985593216/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321097.437013:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321097.437016:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321097.437018:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321097.437021:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6340 00000100:00080000:0.0:1767321097.437023:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321097.437026:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321097.437038:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bea300 x1853163985593344/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321097.437076:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321097.437103:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321097.437104:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321097.437108:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6340 00000100:00080000:0.0:1767321097.437110:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321097.437113:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321097.437124:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9f80 x1853163985593600/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321097.437148:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321097.437152:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321097.437154:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321097.437157:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6340 00000100:00080000:0.0:1767321097.437159:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321097.437179:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321097.437191:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8380 x1853163985593856/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321097.437216:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321097.437232:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985593728:0@lo:400:kworker.0 00000100:00100000:0.0:1767321097.437303:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321097.437518:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985593472:0@lo:400:kworker.0 00000100:00100000:1.0:1767321097.437551:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.437647:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985593216:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.437677:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.437710:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985593344:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.437737:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.437754:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985593600:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.437771:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.437786:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985593856:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.437805:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.437835:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593728 00000100:00100000:2.0:1767321097.437843:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593728 00000100:00100000:2.0:1767321097.437889:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593472 00000100:00100000:2.0:1767321097.437893:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593472 00000100:00100000:2.0:1767321097.437905:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593216 00000100:00100000:2.0:1767321097.437907:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593216 00000100:00100000:2.0:1767321097.437929:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593600 00000100:00100000:2.0:1767321097.437931:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593600 00000100:00100000:2.0:1767321097.437938:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593856 00000100:00100000:2.0:1767321097.437940:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593856 00000100:00100000:2.0:1767321097.437949:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5616 00000100:00100000:2.0:1767321097.437957:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985593728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321097.438003:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985593728:12345-0@lo:400:kworker.0 Request processed in 47us (700us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321097.438010:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5616 00000100:00100000:2.0:1767321097.438026:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5617 00000100:00100000:2.0:1767321097.438032:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985593472:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321097.438063:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985593472:12345-0@lo:400:kworker.0 Request processed in 31us (511us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321097.438068:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5617 00000100:00100000:2.0:1767321097.438091:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6426 00000100:00100000:2.0:1767321097.438095:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985593216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321097.438113:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321097.438144:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985593216:12345-0@lo:38:kworker.0 Request processed in 47us (465us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321097.438150:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6426 00000100:00100000:2.0:1767321097.438181:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6427 00000100:00100000:2.0:1767321097.438186:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985593600:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321097.438195:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321097.438219:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985593600:12345-0@lo:38:kworker.0 Request processed in 32us (446us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321097.438225:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6427 00000100:00100000:2.0:1767321097.438236:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6428 00000100:00100000:2.0:1767321097.438241:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985593856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321097.438250:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321097.438271:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985593856:12345-0@lo:38:kworker.0 Request processed in 29us (465us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321097.438277:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6428 00000100:00080000:2.0:1767321097.438308:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321097.438312:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321097.438315:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985593216:0@lo:38:kworker.0 00000100:00080000:2.0:1767321097.438323:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321097.438326:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321097.438328:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985593600:0@lo:38:kworker.0 00000100:00080000:2.0:1767321097.438337:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321097.438339:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321097.438342:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985593856:0@lo:38:kworker.0 00000100:00100000:1.0:1767321097.438528:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985593472:0@lo:400:kworker.0 00000100:00100000:3.0:1767321097.439280:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593344 00000100:00100000:3.0:1767321097.439287:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593344 00000100:00100000:3.0:1767321097.439323:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2097 00000100:00100000:3.0:1767321097.439328:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190a10050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985593344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321097.439339:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321097.439394:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190a10050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985593344:12345-0@lo:38:kworker.0 Request processed in 64us (1655us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321097.439401:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2097 00000100:00080000:1.0:1767321097.439753:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321097.439758:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321097.439762:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985593344:0@lo:38:kworker.0 00000100:00080000:0.0:1767321097.441006:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321097.441049:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985593728:0@lo:400:kworker.0 00010000:00010000:2.0:1767321098.077252:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321099.099814:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321100.123560:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321101.147868:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321102.043423:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985593984:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321102.043508:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321102.043560:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985593984 00000100:00100000:3.0:1767321102.043569:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985593984 00000100:00080000:1.0:1767321102.043626:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321102.043638:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2854 00000100:00100000:3.0:1767321102.043649:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985593984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321102.043681:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321102.043797:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321102.043809:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985594112:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321102.043835:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321102.043853:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985594112 00000100:00100000:2.0:1767321102.043857:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985594112 00000100:00100000:2.0:1767321102.043877:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2855 00000100:00100000:2.0:1767321102.043881:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985594112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321102.043893:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321102.043913:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985594112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 31us (78us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321102.043918:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2855 00000100:00080000:2.0:1767321102.043934:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321102.043957:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985594112:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321102.054312:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985593984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 10653us (10791us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321102.054328:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2854 00000100:00100000:0.0:1767321102.054393:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985593984:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321102.171485:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321102.511124:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321102.511673:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654464 00000100:00100000:0.0:1767321102.511685:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654464 00000100:00100000:0.0:1767321102.511825:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5618 00000100:00100000:0.0:1767321102.511836:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920654464:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321102.511926:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920654464:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 92us (812us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321102.511935:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5618 00000100:00080000:2.0:1767321102.512564:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321102.514620:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321102.514699:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654336 00000100:00100000:1.0:1767321102.514706:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654336 00000100:00100000:1.0:1767321102.514742:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6429 00000100:00100000:1.0:1767321102.514748:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920654336:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321102.514766:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321102.514821:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920654336:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 72us (206us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321102.514826:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6429 00000100:00080000:3.0:1767321102.561208:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321102.561218:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321102.561221:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321102.561225:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321102.561229:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321102.561237:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6345 00000100:00080000:3.0:1767321102.561242:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321102.561250:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321102.561305:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2300 x1853163985594240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321102.561323:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321102.561326:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321102.561328:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321102.561331:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6345 00000100:00080000:3.0:1767321102.561333:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321102.561336:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321102.561352:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61500 x1853163985594368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321102.561377:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321102.561380:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321102.561381:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321102.561383:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6345 00000100:00080000:3.0:1767321102.561385:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321102.561387:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321102.561393:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61880 x1853163985594624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321102.561408:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321102.561411:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321102.561412:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321102.561415:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6345 00000100:00080000:3.0:1767321102.561417:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321102.561421:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321102.561431:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985594880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321102.561629:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985594496:0@lo:400:kworker.0 00000100:00100000:2.0:1767321102.561688:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321102.561751:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985594752:0@lo:400:kworker.0 00000100:00100000:2.0:1767321102.561765:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321102.561789:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985594496 00000100:00100000:2.0:1767321102.561795:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985594496 00000100:00100000:2.0:1767321102.561844:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985594752 00000100:00100000:2.0:1767321102.561847:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985594752 00000100:00100000:2.0:1767321102.561870:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5619 00000100:00100000:2.0:1767321102.561878:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985594496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321102.561919:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985594496:12345-0@lo:400:kworker.0 Request processed in 43us (231us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321102.561924:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5619 00000100:00100000:2.0:1767321102.561938:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5620 00000100:00100000:2.0:1767321102.561942:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985594752:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321102.561975:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985594752:12345-0@lo:400:kworker.0 Request processed in 31us (207us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321102.561981:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5620 00000100:00080000:2.0:1767321102.562007:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321102.562040:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985594752:0@lo:400:kworker.0 00000100:00100000:2.0:1767321102.562057:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985594496:0@lo:400:kworker.0 00000100:00100000:1.0:1767321102.562861:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985594240:0@lo:38:kworker.0 00000100:00100000:1.0:1767321102.562907:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321102.562994:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985594368:0@lo:38:kworker.0 00000100:00100000:1.0:1767321102.563688:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321102.563779:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985594624:0@lo:38:kworker.0 00000100:00100000:1.0:1767321102.563808:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321102.563923:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985594368 00000100:00100000:2.0:1767321102.563930:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985594368 00000100:00100000:2.0:1767321102.564146:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2098 00000100:00100000:2.0:1767321102.564154:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985594368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321102.564192:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321102.564233:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985594368:12345-0@lo:38:kworker.0 Request processed in 78us (545us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321102.564240:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2098 00000100:00100000:1.0:1767321102.564344:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985594880:0@lo:38:kworker.0 00000100:00100000:1.0:1767321102.564370:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321102.564699:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985594240 00000100:00100000:3.0:1767321102.564705:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985594240 00000100:00100000:3.0:1767321102.564738:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985594624 00000100:00100000:3.0:1767321102.564740:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985594624 00000100:00100000:3.0:1767321102.564758:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6430 00000100:00100000:3.0:1767321102.564763:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985594240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321102.564774:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321102.564807:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985594240:12345-0@lo:38:kworker.0 Request processed in 43us (1899us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321102.564813:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6430 00000100:00100000:3.0:1767321102.564826:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6431 00000100:00100000:3.0:1767321102.564830:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985594624:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321102.564839:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321102.564864:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985594624:12345-0@lo:38:kworker.0 Request processed in 33us (1055us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321102.564870:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6431 00000100:00080000:1.0:1767321102.565050:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321102.565057:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321102.565061:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985594240:0@lo:38:kworker.0 00000100:00080000:1.0:1767321102.565072:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321102.565075:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321102.565078:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985594368:0@lo:38:kworker.0 00000100:00080000:1.0:1767321102.565086:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321102.565089:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321102.565091:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985594624:0@lo:38:kworker.0 00000100:00100000:0.0:1767321102.565574:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985594880 00000100:00100000:0.0:1767321102.565579:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985594880 00000100:00100000:2.0:1767321102.565636:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6432 00000100:00100000:2.0:1767321102.565641:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985594880:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321102.565654:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321102.565702:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985594880:12345-0@lo:38:kworker.0 Request processed in 59us (1330us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321102.565708:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6432 00000100:00080000:1.0:1767321102.565780:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321102.565785:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321102.565788:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985594880:0@lo:38:kworker.0 00010000:00010000:2.0:1767321103.195516:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321104.219518:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321105.243698:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321106.268404:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321107.163521:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985595008:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321107.163605:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.164307:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321107.164321:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985595136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321107.164361:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321107.164447:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595008 00000100:00100000:1.0:1767321107.164455:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595008 00000100:00100000:1.0:1767321107.164515:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595136 00000100:00100000:1.0:1767321107.164518:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595136 00000100:00100000:1.0:1767321107.164540:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2856 00000100:00100000:1.0:1767321107.164550:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985595008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321107.164579:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321107.164638:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985595008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (1031us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321107.164646:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2856 00000100:00100000:1.0:1767321107.164664:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2857 00000100:00100000:1.0:1767321107.164670:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985595136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321107.164685:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321107.164712:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985595136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 42us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321107.164718:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2857 00000100:00100000:0.0:1767321107.164947:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985595008:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321107.165092:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321107.165097:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321107.165128:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985595136:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321107.291436:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321107.626317:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321107.626370:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321107.626438:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654592 00000100:00100000:0.0:1767321107.626454:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654592 00000100:00100000:0.0:1767321107.626515:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654720 00000100:00100000:0.0:1767321107.626518:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654720 00000100:00100000:0.0:1767321107.626589:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5621 00000100:00100000:0.0:1767321107.626603:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920654720:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321107.626693:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920654720:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 95us (322us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321107.626701:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5621 00000100:00100000:0.0:1767321107.626720:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6433 00000100:00100000:0.0:1767321107.626724:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920654592:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321107.626745:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321107.626771:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920654592:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (472us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321107.626778:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6433 00000100:00080000:1.0:1767321107.627290:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321107.675791:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321107.675801:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321107.675805:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321107.675809:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321107.675812:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321107.675821:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6350 00000100:00080000:3.0:1767321107.675826:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321107.675833:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321107.675868:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985595264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321107.675951:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985595264:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.676023:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321107.676240:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321107.676245:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321107.676247:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321107.676251:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6350 00000100:00080000:3.0:1767321107.676255:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321107.676259:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321107.676290:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3480 x1853163985595392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321107.680984:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985595392:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.681076:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.681184:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595264 00000100:00100000:2.0:1767321107.681194:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595264 00000100:00100000:2.0:1767321107.681246:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6434 00000100:00100000:2.0:1767321107.681254:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985595264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321107.681288:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321107.681334:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985595264:12345-0@lo:38:kworker.0 Request processed in 80us (5311us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321107.681341:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6434 00000100:00100000:2.0:1767321107.681365:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595392 00000100:00100000:2.0:1767321107.681368:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595392 00000100:00080000:0.0:1767321107.681847:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321107.681853:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321107.681858:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985595264:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.682097:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2099 00000100:00100000:2.0:1767321107.682103:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985595392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321107.682117:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321107.682190:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985595392:12345-0@lo:38:kworker.0 Request processed in 85us (1113us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321107.682197:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2099 00000100:00080000:1.0:1767321107.682266:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321107.682283:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321107.682286:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985595392:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.685033:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985595520:0@lo:400:kworker.0 00000100:00100000:2.0:1767321107.685144:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321107.685386:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321107.685392:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321107.685397:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321107.685404:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6350 00000100:00080000:3.0:1767321107.685409:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321107.685417:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321107.685485:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1180 x1853163985595648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321107.685554:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321107.685558:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321107.685559:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321107.685563:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6350 00000100:00080000:3.0:1767321107.685566:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321107.685570:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321107.685580:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2300 x1853163985595904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321107.685619:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321107.685635:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985595776:0@lo:400:kworker.0 00000100:00100000:3.0:1767321107.685673:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321107.686179:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985595648:0@lo:38:kworker.0 00000100:00100000:0.0:1767321107.687381:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595520 00000100:00100000:0.0:1767321107.687400:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595520 00000100:00100000:0.0:1767321107.687498:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595776 00000100:00100000:0.0:1767321107.687502:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595776 00000100:00100000:0.0:1767321107.687679:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5622 00000100:00100000:0.0:1767321107.687692:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985595520:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321107.687971:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321107.688004:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5623 00000100:00100000:1.0:1767321107.688032:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985595776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321107.688044:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321107.688084:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985595904:0@lo:38:kworker.0 00000100:00100000:3.0:1767321107.688137:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321107.688160:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985595776:12345-0@lo:400:kworker.0 Request processed in 141us (2482us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321107.688202:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5623 00000100:00100000:3.0:1767321107.688219:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595648 00000100:00100000:3.0:1767321107.688227:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595648 00000100:00100000:1.0:1767321107.688248:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985595904 00000100:00100000:1.0:1767321107.688257:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985595904 00000100:00100000:3.0:1767321107.688273:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6435 00000100:00100000:3.0:1767321107.688281:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985595648:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321107.688308:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321107.688309:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6436 00000100:00100000:1.0:1767321107.688323:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985595904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321107.688348:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321107.688363:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985595648:12345-0@lo:38:kworker.0 Request processed in 79us (318us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321107.688372:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6435 00000100:00100000:1.0:1767321107.688406:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985595904:12345-0@lo:38:kworker.0 Request processed in 80us (265us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321107.688416:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6436 00000100:00080000:3.0:1767321107.688434:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321107.688443:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321107.688451:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985595648:0@lo:38:kworker.0 00000100:00080000:3.0:1767321107.688480:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321107.688484:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321107.688489:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985595904:0@lo:38:kworker.0 00000100:00100000:1.0:1767321107.688495:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985595776:0@lo:400:kworker.0 00000100:00100000:0.0:1767321107.688963:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985595520:12345-0@lo:400:kworker.0 Request processed in 1185us (3730us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321107.688974:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5622 00000100:00100000:2.0:1767321107.689145:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985595520:0@lo:400:kworker.0 00010000:00010000:2.0:1767321108.315468:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321109.339599:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321110.371480:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321111.390956:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321112.283798:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985596032:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321112.283897:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321112.284062:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596032 00000100:00100000:1.0:1767321112.284071:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596032 00000100:00100000:1.0:1767321112.284140:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2858 00000100:00100000:1.0:1767321112.284150:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985596032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321112.284207:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321112.284268:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985596032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 119us (371us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321112.284277:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2858 00000100:00080000:3.0:1767321112.284377:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321112.284417:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985596032:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321112.286583:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985596160:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321112.286646:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321112.286758:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596160 00000100:00100000:1.0:1767321112.286766:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596160 00000100:00100000:1.0:1767321112.286836:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2859 00000100:00100000:1.0:1767321112.286845:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985596160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321112.286875:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321112.286918:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985596160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (272us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321112.286927:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2859 00000100:00080000:3.0:1767321112.286994:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321112.287061:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985596160:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321112.411656:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321112.745556:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321112.745599:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321112.745710:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654976 00000100:00100000:0.0:1767321112.745720:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654976 00000100:00100000:0.0:1767321112.745781:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920654848 00000100:00100000:0.0:1767321112.745785:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920654848 00000100:00100000:0.0:1767321112.745813:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5624 00000100:00100000:0.0:1767321112.745823:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920654976:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321112.745896:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920654976:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (349us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321112.745903:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5624 00000100:00100000:0.0:1767321112.745922:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6437 00000100:00100000:0.0:1767321112.745927:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920654848:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321112.745946:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321112.745971:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920654848:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (371us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321112.745978:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6437 00000100:00080000:2.0:1767321112.747392:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321112.795402:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321112.795412:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321112.795416:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321112.795420:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321112.795424:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321112.795432:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6355 00000100:00080000:3.0:1767321112.795437:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321112.795443:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321112.795486:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1f80 x1853163985596288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321112.795503:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321112.795506:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321112.795507:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321112.795511:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6355 00000100:00080000:3.0:1767321112.795513:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321112.795517:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321112.795529:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0380 x1853163985596416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321112.795570:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321112.795574:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321112.795576:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321112.795579:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6355 00000100:00080000:3.0:1767321112.795581:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321112.795584:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321112.795596:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985596672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321112.795621:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321112.795625:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321112.795627:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321112.795630:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6355 00000100:00080000:3.0:1767321112.795632:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321112.795634:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321112.795643:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7b80 x1853163985596928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321112.795681:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985596800:0@lo:400:kworker.0 00000100:00100000:3.0:1767321112.795748:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321112.795821:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321112.795827:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985597056:0@lo:400:kworker.0 00000100:00100000:3.0:1767321112.795852:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321112.795882:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596800 00000100:00100000:3.0:1767321112.795889:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596800 00000100:00100000:3.0:1767321112.795939:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5625 00000100:00100000:3.0:1767321112.795947:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985596800:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321112.795993:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985596800:12345-0@lo:400:kworker.0 Request processed in 46us (244us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321112.796000:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5625 00000100:00080000:3.0:1767321112.796033:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321112.796061:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985596800:0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796301:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985596544:0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796332:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321112.796359:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321112.796365:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985597184:0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796385:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321112.796409:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597056 00000100:00100000:2.0:1767321112.796414:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597056 00000100:00100000:2.0:1767321112.796444:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597184 00000100:00100000:2.0:1767321112.796447:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597184 00000100:00100000:2.0:1767321112.796464:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3635 00000100:00100000:2.0:1767321112.796470:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985597056:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796505:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985597056:12345-0@lo:400:kworker.0 Request processed in 35us (651us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321112.796511:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3635 00000100:00100000:2.0:1767321112.796524:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3636 00000100:00100000:2.0:1767321112.796530:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985597184:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796559:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985597184:12345-0@lo:400:kworker.0 Request processed in 28us (171us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321112.796564:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3636 00000100:00100000:2.0:1767321112.796589:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985597184:0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796603:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596544 00000100:00100000:2.0:1767321112.796606:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596544 00000100:00100000:2.0:1767321112.796633:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5626 00000100:00100000:2.0:1767321112.796638:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985596544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796668:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985596544:12345-0@lo:400:kworker.0 Request processed in 30us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321112.796673:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5626 00000100:00080000:2.0:1767321112.796689:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321112.796702:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985596544:0@lo:400:kworker.0 00000100:00100000:2.0:1767321112.796723:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985597056:0@lo:400:kworker.0 00000100:00100000:0.0:1767321112.797436:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985596288:0@lo:38:kworker.0 00000100:00100000:0.0:1767321112.797474:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321112.797511:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985596416:0@lo:38:kworker.0 00000100:00100000:0.0:1767321112.797535:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321112.797567:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985596672:0@lo:38:kworker.0 00000100:00100000:0.0:1767321112.797587:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321112.797612:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985596928:0@lo:38:kworker.0 00000100:00100000:0.0:1767321112.797635:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321112.798570:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596416 00000100:00100000:2.0:1767321112.798579:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596416 00000100:00100000:2.0:1767321112.798627:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2100 00000100:00100000:2.0:1767321112.798633:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985596416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321112.798663:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321112.798716:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985596416:12345-0@lo:38:kworker.0 Request processed in 81us (1180us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321112.798723:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2100 00000100:00100000:2.0:1767321112.798745:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596288 00000100:00100000:2.0:1767321112.798749:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596288 00000100:00100000:2.0:1767321112.798763:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596672 00000100:00100000:2.0:1767321112.798765:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596672 00000100:00100000:2.0:1767321112.798776:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985596928 00000100:00100000:2.0:1767321112.798779:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985596928 00000100:00100000:2.0:1767321112.798791:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6438 00000100:00100000:2.0:1767321112.798807:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985596288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321112.798817:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321112.798840:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985596288:12345-0@lo:38:kworker.0 Request processed in 44us (1366us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321112.798845:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6438 00000100:00100000:2.0:1767321112.798855:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6439 00000100:00100000:2.0:1767321112.798858:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985596672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321112.798866:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321112.798881:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985596672:12345-0@lo:38:kworker.0 Request processed in 22us (1294us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321112.798887:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6439 00000100:00100000:2.0:1767321112.798894:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6440 00000100:00100000:2.0:1767321112.798897:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985596928:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321112.798905:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321112.798925:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985596928:12345-0@lo:38:kworker.0 Request processed in 26us (1289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321112.798930:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6440 00000100:00080000:0.0:1767321112.801383:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321112.801409:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321112.801415:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985596288:0@lo:38:kworker.0 00000100:00080000:0.0:1767321112.801429:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321112.801432:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321112.801435:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985596416:0@lo:38:kworker.0 00000100:00080000:0.0:1767321112.801446:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321112.801449:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321112.801452:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985596672:0@lo:38:kworker.0 00000100:00080000:0.0:1767321112.801461:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321112.801464:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321112.801467:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985596928:0@lo:38:kworker.0 00010000:00010000:1.0:1767321113.438386:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321114.460099:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321115.483403:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321116.507543:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321117.404048:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985597312:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321117.404119:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321117.404496:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597312 00000100:00100000:1.0:1767321117.404505:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597312 00000100:00100000:1.0:1767321117.404580:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2860 00000100:00100000:1.0:1767321117.404589:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985597312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321117.404619:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321117.404671:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985597312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (552us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321117.404678:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2860 00000100:00100000:1.0:1767321117.404730:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985597312:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321117.404875:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321117.408764:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321117.408791:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985597440:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321117.408865:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321117.410010:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597440 00000100:00100000:1.0:1767321117.410022:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597440 00000100:00100000:1.0:1767321117.410118:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2861 00000100:00100000:1.0:1767321117.410129:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985597440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321117.410158:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767321117.410242:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321117.411400:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985597440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1268us (2529us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321117.411411:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2861 00000100:00100000:0.0:1767321117.411549:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985597440:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321117.531938:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321117.866312:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767321117.866369:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920655104 offset 0 length 520: 4 00000100:00100000:3.0:1767321117.866410:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321117.866631:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920655360 00000100:00100000:0.0:1767321117.866651:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920655360 00000100:00100000:0.0:1767321117.866740:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920655232 00000100:00100000:0.0:1767321117.866744:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920655232 00000100:00100000:0.0:1767321117.866769:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5627 00000100:00100000:0.0:1767321117.866781:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920655360:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321117.866895:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920655360:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 116us (601us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321117.866905:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5627 00000100:00100000:0.0:1767321117.866928:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6441 00000100:00100000:0.0:1767321117.866932:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920655232:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321117.866956:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321117.866988:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920655232:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (588us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321117.866999:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6441 00000100:00080000:3.0:1767321117.867598:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321117.915368:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321092/real 1767321092] req@ffff8ce1b3ff1c00 x1853163985592064/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321117 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321117.915370:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.915388:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321117.915406:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.915412:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.915417:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767321117.915424:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3ff1c00 x1853163985592064/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321117 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321117.915431:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.915438:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.915448:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00080000:2.0:1767321117.915473:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767321117.915516:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593b800 x1853163985597568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321117.915538:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.915542:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.915544:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.915549:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.915552:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.915556:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321117.915575:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938000 x1853163985597696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321117.915582:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321117.915598:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985592064:0@lo:250:kworker.0 00000100:00100000:2.0:1767321117.915636:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985597568:0@lo:38:kworker.0 00000100:00080000:1.0:1767321117.915646:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.915653:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.915655:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.915660:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.915663:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.915667:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321117.915694:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5939180 x1853163985597952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321117.915727:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321117.915781:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.915785:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.915787:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.915792:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.915794:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.915799:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321117.915805:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985597696:0@lo:38:kworker.0 00000100:00100000:1.0:1767321117.915817:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938700 x1853163985598208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321117.915841:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321117.916025:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597568 00000100:00100000:3.0:1767321117.916035:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597568 00000100:00100000:3.0:1767321117.916072:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6442 00000100:00100000:3.0:1767321117.916080:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985597568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321117.916101:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321117.916141:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985597568:12345-0@lo:38:kworker.0 Request processed in 60us (415us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321117.916149:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6442 00000100:00100000:3.0:1767321117.916201:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597696 00000100:00100000:3.0:1767321117.916205:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597696 00000100:00100000:3.0:1767321117.916224:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2101 00000100:00100000:3.0:1767321117.916228:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985597696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321117.916238:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321117.916263:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985597696:12345-0@lo:38:kworker.0 Request processed in 33us (425us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321117.916269:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2101 00000100:00100000:3.0:1767321117.916316:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985598464:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916344:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321117.916367:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321117.916373:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985597824:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916394:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321117.916418:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597824 00000100:00100000:3.0:1767321117.916422:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597824 00000100:00100000:3.0:1767321117.916459:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5628 00000100:00100000:3.0:1767321117.916466:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985597824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916505:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985597824:12345-0@lo:400:kworker.0 Request processed in 40us (110us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321117.916512:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5628 00000100:00080000:3.0:1767321117.916529:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321117.916556:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985597824:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916593:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985598464 00000100:00100000:3.0:1767321117.916596:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985598464 00000100:00100000:3.0:1767321117.916624:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3637 00000100:00100000:3.0:1767321117.916630:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985598464:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916662:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985598464:12345-0@lo:400:kworker.0 Request processed in 33us (317us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321117.916668:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3637 00000100:00100000:3.0:1767321117.916695:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985598464:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916726:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985598336:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916750:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321117.916768:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321117.916773:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985598080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916792:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321117.916813:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985598336 00000100:00100000:3.0:1767321117.916816:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985598336 00000100:00100000:3.0:1767321117.916836:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3638 00000100:00100000:3.0:1767321117.916842:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985598336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916874:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985598336:12345-0@lo:400:kworker.0 Request processed in 32us (123us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321117.916880:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3638 00000100:00100000:3.0:1767321117.916895:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985598080 00000100:00100000:3.0:1767321117.916898:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985598080 00000100:00100000:3.0:1767321117.916923:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5629 00000100:00100000:3.0:1767321117.916929:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985598080:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.916957:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985598080:12345-0@lo:400:kworker.0 Request processed in 29us (164us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321117.916964:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5629 00000100:00080000:3.0:1767321117.916976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321117.916995:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985598080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.917005:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985598336:0@lo:400:kworker.0 00000100:00080000:2.0:1767321117.917113:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.917119:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321117.917122:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985597696:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.917148:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985597952:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.917189:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321117.917212:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985598208:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.917232:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321117.917253:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.917257:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321117.917260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985597568:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.917283:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985597952 00000100:00100000:2.0:1767321117.917286:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985597952 00000100:00100000:2.0:1767321117.917305:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985598208 00000100:00100000:2.0:1767321117.917307:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985598208 00000100:00100000:2.0:1767321117.917322:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6443 00000100:00100000:2.0:1767321117.917326:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985597952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321117.917337:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321117.917366:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985597952:12345-0@lo:38:kworker.0 Request processed in 39us (177us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321117.917372:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6443 00000100:00100000:2.0:1767321117.917386:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6444 00000100:00100000:2.0:1767321117.917389:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985598208:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321117.917398:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321117.917421:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985598208:12345-0@lo:38:kworker.0 Request processed in 30us (188us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321117.917427:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6444 00000100:00080000:2.0:1767321117.917456:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.917460:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321117.917463:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985597952:0@lo:38:kworker.0 00000100:00080000:2.0:1767321117.917471:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.917473:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321117.917476:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985598208:0@lo:38:kworker.0 00000100:00080000:1.0:1767321117.919940:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.921554:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767321117.921560:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.921572:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6340 00000100:00080000:1.0:1767321117.921578:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.921587:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321117.922264:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a593a680 x1853163985598592/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321117.922408:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985598592:0@lo:250:kworker.0 00000400:00000100:2.0:1767321117.922482:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985598592 offset 0 length 520: 4 00010000:00010000:0.0:1767321118.556197:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321119.582598:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321120.603407:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321121.629580:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321122.526300:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321122.526324:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985598720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321122.526388:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321122.526917:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985598720 00000100:00100000:1.0:1767321122.526926:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985598720 00000100:00100000:1.0:1767321122.527015:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2862 00000100:00100000:1.0:1767321122.527025:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985598720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321122.527055:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321122.527106:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985598720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (717us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321122.527114:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2862 00000100:00080000:3.0:1767321122.527204:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321122.527531:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985598848:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321122.527563:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321122.528316:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985598848 00000100:00100000:0.0:1767321122.528324:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985598848 00000100:00100000:0.0:1767321122.528382:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2863 00000100:00100000:0.0:1767321122.528391:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985598848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321122.528411:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321122.528457:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985598848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (892us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321122.528464:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2863 00000100:00080000:3.0:1767321122.528534:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321122.528573:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985598848:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321122.529017:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985598720:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321122.651777:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321122.996513:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321122.996634:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321122.996768:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920655616 00000100:00100000:0.0:1767321122.996778:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920655616 00000100:00100000:0.0:1767321122.996850:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920655488 00000100:00100000:0.0:1767321122.996854:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920655488 00000100:00100000:0.0:1767321122.996873:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5630 00000100:00100000:0.0:1767321122.996883:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920655616:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321122.996951:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920655616:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 69us (447us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321122.996958:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5630 00000100:00100000:0.0:1767321122.996976:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6445 00000100:00100000:0.0:1767321122.996981:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920655488:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321122.997001:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321122.997043:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920655488:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 60us (436us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321122.997050:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6445 00000100:00080000:3.0:1767321122.997932:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321123.035513:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321123.035524:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321123.035527:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321123.035530:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321123.035534:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321123.035542:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6366 00000100:00080000:0.0:1767321123.035547:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321123.035554:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321123.035597:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8380 x1853163985598976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321123.035631:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321123.035634:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321123.035636:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321123.035639:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6366 00000100:00080000:0.0:1767321123.035642:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321123.035646:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321123.035661:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9f80 x1853163985599104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321123.035702:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321123.035706:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321123.035708:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321123.035712:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6366 00000100:00080000:0.0:1767321123.035714:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321123.035717:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321123.035729:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8700 x1853163985599360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321123.035754:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321123.035758:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321123.035759:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321123.035762:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6366 00000100:00080000:0.0:1767321123.035764:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321123.035767:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321123.035777:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985599616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321123.035933:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985599232:0@lo:400:kworker.0 00000100:00100000:1.0:1767321123.036002:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321123.036075:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321123.036081:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985599872:0@lo:400:kworker.0 00000100:00100000:1.0:1767321123.036106:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321123.036195:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985599232 00000100:00100000:0.0:1767321123.036202:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985599232 00000100:00100000:0.0:1767321123.036272:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5631 00000100:00100000:0.0:1767321123.036281:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985599232:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321123.039941:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985599232:12345-0@lo:400:kworker.0 Request processed in 3650us (3927us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321123.039956:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5631 00000100:00080000:1.0:1767321123.040089:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321123.040140:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985599232:0@lo:400:kworker.0 00000100:00100000:3.0:1767321123.040624:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985599872 00000100:00100000:3.0:1767321123.040633:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985599872 00000100:00100000:3.0:1767321123.040688:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3639 00000100:00100000:3.0:1767321123.040696:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985599872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321123.040783:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985599872:12345-0@lo:400:kworker.0 Request processed in 88us (4675us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321123.040790:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3639 00000100:00100000:3.0:1767321123.040818:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985599744:0@lo:400:kworker.0 00000100:00100000:3.0:1767321123.040855:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321123.040890:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321123.040897:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985599488:0@lo:400:kworker.0 00000100:00100000:3.0:1767321123.040922:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321123.040959:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985599744 00000100:00100000:3.0:1767321123.040963:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985599744 00000100:00100000:3.0:1767321123.040984:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3640 00000100:00100000:3.0:1767321123.040990:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a01c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985599744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321123.041023:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a01c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985599744:12345-0@lo:400:kworker.0 Request processed in 32us (167us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321123.041028:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3640 00000100:00100000:3.0:1767321123.041396:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985599744:0@lo:400:kworker.0 00000100:00100000:1.0:1767321123.041798:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985599872:0@lo:400:kworker.0 00000100:00100000:2.0:1767321123.042022:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985598976:0@lo:38:kworker.0 00000100:00100000:2.0:1767321123.042067:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321123.042105:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985599104:0@lo:38:kworker.0 00000100:00100000:2.0:1767321123.042141:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321123.042187:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985599360:0@lo:38:kworker.0 00000100:00100000:2.0:1767321123.042210:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321123.042239:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985599616:0@lo:38:kworker.0 00000100:00100000:2.0:1767321123.042259:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321123.042294:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985599488 00000100:00100000:2.0:1767321123.042300:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985599488 00000100:00100000:2.0:1767321123.042344:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985598976 00000100:00100000:2.0:1767321123.042348:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985598976 00000100:00100000:2.0:1767321123.042355:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985599360 00000100:00100000:2.0:1767321123.042357:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985599360 00000100:00100000:2.0:1767321123.042364:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985599616 00000100:00100000:2.0:1767321123.042366:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985599616 00000100:00100000:2.0:1767321123.042375:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5632 00000100:00100000:2.0:1767321123.042382:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985599488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321123.042424:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985599488:12345-0@lo:400:kworker.0 Request processed in 42us (1501us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321123.042431:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5632 00000100:00100000:2.0:1767321123.042448:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6446 00000100:00100000:2.0:1767321123.042464:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985598976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321123.042482:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321123.042514:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985598976:12345-0@lo:38:kworker.0 Request processed in 61us (447us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321123.042520:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6446 00000100:00100000:2.0:1767321123.042531:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6447 00000100:00100000:2.0:1767321123.042534:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985599360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321123.042543:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321123.042566:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985599360:12345-0@lo:38:kworker.0 Request processed in 29us (354us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321123.042571:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6447 00000100:00100000:2.0:1767321123.042581:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6448 00000100:00100000:2.0:1767321123.042584:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985599616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321123.042593:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321123.042626:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985599616:12345-0@lo:38:kworker.0 Request processed in 40us (365us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321123.042632:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6448 00000100:00080000:2.0:1767321123.042656:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321123.042678:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985599488:0@lo:400:kworker.0 00000100:00080000:2.0:1767321123.042706:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321123.042711:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321123.042715:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985598976:0@lo:38:kworker.0 00000100:00080000:2.0:1767321123.042722:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321123.042725:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321123.042728:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985599360:0@lo:38:kworker.0 00000100:00080000:2.0:1767321123.042735:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321123.042738:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321123.042740:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985599616:0@lo:38:kworker.0 00000100:00100000:3.0:1767321123.043369:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985599104 00000100:00100000:3.0:1767321123.043376:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985599104 00000100:00100000:3.0:1767321123.043412:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2102 00000100:00100000:3.0:1767321123.043418:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985599104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321123.043429:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321123.043488:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985599104:12345-0@lo:38:kworker.0 Request processed in 69us (1346us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321123.043494:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2102 00000100:00080000:2.0:1767321123.043678:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321123.043683:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321123.043686:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985599104:0@lo:38:kworker.0 00010000:00010000:2.0:1767321123.675419:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321124.699430:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321125.723729:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321126.747415:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321127.644355:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985600000:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321127.644432:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321127.644706:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600000 00000100:00100000:1.0:1767321127.644716:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600000 00000100:00100000:1.0:1767321127.644784:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2864 00000100:00100000:1.0:1767321127.644793:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985600000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321127.644825:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321127.644882:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985600000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (451us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321127.644891:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2864 00000100:00100000:2.0:1767321127.645038:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985600000:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321127.648562:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321127.651102:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985600128:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321127.651229:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321127.651334:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600128 00000100:00100000:1.0:1767321127.651342:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600128 00000100:00100000:1.0:1767321127.651404:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2865 00000100:00100000:1.0:1767321127.651414:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985600128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321127.651443:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767321127.651531:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321127.656693:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985600128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5270us (5454us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321127.656708:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2865 00000100:00100000:2.0:1767321127.656995:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985600128:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321127.774226:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321128.107987:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321128.108036:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321128.108136:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920655872 00000100:00100000:2.0:1767321128.108146:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920655872 00000100:00100000:2.0:1767321128.108258:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920655744 00000100:00100000:2.0:1767321128.108264:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920655744 00000100:00100000:2.0:1767321128.108284:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5633 00000100:00100000:2.0:1767321128.108295:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920655872:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321128.108372:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920655872:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (395us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321128.108380:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5633 00000100:00100000:2.0:1767321128.108401:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6449 00000100:00100000:2.0:1767321128.108405:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920655744:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321128.108425:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321128.108450:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920655744:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (413us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321128.108456:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6449 00000100:00080000:3.0:1767321128.109237:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321128.157736:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321128.157746:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321128.157749:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321128.157753:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321128.157759:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321128.157767:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6371 00000100:00080000:0.0:1767321128.157773:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321128.157780:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321128.157822:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351ad80 x1853163985600256/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321128.157848:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321128.157852:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321128.157854:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321128.157858:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6371 00000100:00080000:0.0:1767321128.157860:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321128.157864:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321128.157876:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351b800 x1853163985600384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321128.157927:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321128.157931:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321128.157933:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321128.157937:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6371 00000100:00080000:0.0:1767321128.157939:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321128.157943:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321128.157954:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3519880 x1853163985600640/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321128.157983:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321128.157986:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321128.157988:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321128.157992:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6371 00000100:00080000:0.0:1767321128.157994:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321128.157998:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321128.158010:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b351a300 x1853163985600896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321128.158015:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321128.158037:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985600512:0@lo:400:kworker.0 00000100:00100000:0.0:1767321128.158058:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985601024:0@lo:400:kworker.0 00000100:00100000:3.0:1767321128.158149:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321128.158159:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985601152:0@lo:400:kworker.0 00000100:00100000:2.0:1767321128.158281:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985600256:0@lo:38:kworker.0 00000100:00100000:1.0:1767321128.160121:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321128.160222:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600512 00000100:00100000:1.0:1767321128.160231:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600512 00000100:00100000:1.0:1767321128.160275:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5634 00000100:00100000:1.0:1767321128.160281:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985600512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321128.160330:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985600512:12345-0@lo:400:kworker.0 Request processed in 50us (214us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321128.160335:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5634 00000100:00100000:1.0:1767321128.160365:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985600512:0@lo:400:kworker.0 00000100:00100000:1.0:1767321128.160402:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985600768:0@lo:400:kworker.0 00000100:00100000:1.0:1767321128.160429:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321128.160457:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600768 00000100:00100000:1.0:1767321128.160460:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600768 00000100:00100000:1.0:1767321128.160483:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5635 00000100:00100000:1.0:1767321128.160489:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985600768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321128.160535:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985600768:12345-0@lo:400:kworker.0 Request processed in 46us (104us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321128.160541:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5635 00000100:00100000:1.0:1767321128.160664:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985600768:0@lo:400:kworker.0 00000100:00080000:1.0:1767321128.160684:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321128.162918:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321128.163016:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601152 00000100:00100000:1.0:1767321128.163025:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601152 00000100:00100000:1.0:1767321128.163076:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3641 00000100:00100000:1.0:1767321128.163084:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985601152:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321128.163151:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985601152:12345-0@lo:400:kworker.0 Request processed in 67us (239us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321128.163158:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3641 00000100:00100000:3.0:1767321128.163197:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985601152:0@lo:400:kworker.0 00000100:00100000:0.0:1767321128.172623:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321128.172693:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601024 00000100:00100000:1.0:1767321128.172703:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601024 00000100:00100000:1.0:1767321128.172763:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3642 00000100:00100000:1.0:1767321128.172774:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985601024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321128.172843:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985601024:12345-0@lo:400:kworker.0 Request processed in 72us (230us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321128.172851:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3642 00000100:00100000:0.0:1767321128.173237:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985601024:0@lo:400:kworker.0 00000100:00100000:2.0:1767321128.174262:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321128.174317:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600256 00000100:00100000:2.0:1767321128.174317:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985600384:0@lo:38:kworker.0 00000100:00100000:1.0:1767321128.174323:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600256 00000100:00100000:1.0:1767321128.174360:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6450 00000100:00100000:1.0:1767321128.174367:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985600256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321128.174396:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321128.174447:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985600256:12345-0@lo:38:kworker.0 Request processed in 79us (187us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321128.174456:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321128.174460:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6450 00000100:00100000:2.0:1767321128.174495:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985600640:0@lo:38:kworker.0 00000100:00100000:3.0:1767321128.174605:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600384 00000100:00100000:3.0:1767321128.174610:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600384 00000100:00100000:3.0:1767321128.174642:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2103 00000100:00100000:3.0:1767321128.174647:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985600384:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321128.174658:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321128.174688:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985600384:12345-0@lo:38:kworker.0 Request processed in 39us (236us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321128.174693:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2103 00000100:00100000:2.0:1767321128.175934:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321128.175982:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985600896:0@lo:38:kworker.0 00000100:00100000:2.0:1767321128.176022:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321128.176072:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321128.176080:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321128.176085:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985600256:0@lo:38:kworker.0 00000100:00100000:3.0:1767321128.176091:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600640 00000100:00100000:3.0:1767321128.176098:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600640 00000100:00080000:2.0:1767321128.176099:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321128.176104:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321128.176108:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985600384:0@lo:38:kworker.0 00000100:00100000:3.0:1767321128.176130:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985600896 00000100:00100000:3.0:1767321128.176134:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985600896 00000100:00100000:3.0:1767321128.176186:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6451 00000100:00100000:3.0:1767321128.176192:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985600640:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321128.176209:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321128.176248:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985600640:12345-0@lo:38:kworker.0 Request processed in 54us (1719us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321128.176256:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6451 00000100:00100000:3.0:1767321128.176273:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6452 00000100:00100000:3.0:1767321128.176277:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985600896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321128.176286:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321128.176308:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985600896:12345-0@lo:38:kworker.0 Request processed in 29us (285us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321128.176313:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6452 00000100:00080000:3.0:1767321128.179113:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321128.179125:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321128.179130:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985600640:0@lo:38:kworker.0 00000100:00080000:3.0:1767321128.179146:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321128.179148:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321128.179150:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985600896:0@lo:38:kworker.0 00010000:00010000:2.0:1767321128.796728:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321129.822696:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321130.843863:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321131.867413:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321132.763500:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985601280:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321132.763569:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321132.763777:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321132.763788:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985601408:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321132.763818:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321132.763841:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601280 00000100:00100000:1.0:1767321132.763851:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601280 00000100:00100000:1.0:1767321132.763907:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601408 00000100:00100000:1.0:1767321132.763910:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601408 00000100:00100000:1.0:1767321132.763932:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2866 00000100:00100000:1.0:1767321132.763941:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985601280:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321132.763965:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767321132.763974:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321132.764020:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985601280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (450us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321132.764027:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2866 00000100:00100000:1.0:1767321132.764042:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2867 00000100:00100000:1.0:1767321132.764047:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985601408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321132.764062:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321132.764089:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985601408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (270us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321132.764094:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2867 00000100:00080000:0.0:1767321132.764107:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321132.764144:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985601408:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321132.769439:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985601280:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321132.897912:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321133.226136:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321133.228774:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920656128 00000100:00100000:2.0:1767321133.228790:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920656128 00000100:00100000:2.0:1767321133.228874:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5636 00000100:00100000:2.0:1767321133.228890:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920656128:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321133.229001:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920656128:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 118us (2875us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321133.229009:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5636 00000100:00080000:0.0:1767321133.229104:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321133.231432:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321133.232657:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920656000 00000100:00100000:3.0:1767321133.232670:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920656000 00000100:00100000:3.0:1767321133.232734:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6453 00000100:00100000:3.0:1767321133.232742:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920656000:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321133.232778:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321133.232857:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920656000:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 114us (1452us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321133.232865:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6453 00000100:00080000:3.0:1767321133.280670:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321133.280681:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321133.280683:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321133.280688:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321133.280693:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321133.280701:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6376 00000100:00080000:3.0:1767321133.280706:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321133.280714:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321133.280838:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6d80 x1853163985601536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321133.280939:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985601536:0@lo:38:kworker.0 00000100:00100000:2.0:1767321133.281006:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321133.281187:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321133.281192:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321133.281193:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321133.281197:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6376 00000100:00080000:3.0:1767321133.281200:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321133.281204:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321133.281223:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7480 x1853163985601664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321133.287613:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601536 00000100:00100000:0.0:1767321133.287632:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601536 00000100:00100000:0.0:1767321133.287753:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6454 00000100:00100000:0.0:1767321133.287762:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985601536:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321133.287789:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321133.291288:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985601664:0@lo:38:kworker.0 00000100:00100000:2.0:1767321133.291370:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321133.291467:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601664 00000100:00100000:1.0:1767321133.291478:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601664 00000100:00100000:1.0:1767321133.291516:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2104 00000100:00100000:1.0:1767321133.291523:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985601664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321133.291544:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321133.291588:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985601664:12345-0@lo:38:kworker.0 Request processed in 64us (217us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321133.291596:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2104 00000100:00100000:0.0:1767321133.296677:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985601536:12345-0@lo:38:kworker.0 Request processed in 8906us (15661us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321133.296692:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6454 00000100:00080000:2.0:1767321133.297893:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321133.297902:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321133.297908:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985601536:0@lo:38:kworker.0 00000100:00080000:2.0:1767321133.297947:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321133.297951:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321133.297954:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985601664:0@lo:38:kworker.0 00000100:00080000:3.0:1767321133.308661:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321133.308669:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321133.308673:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321133.308682:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6376 00000100:00080000:3.0:1767321133.308687:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321133.308694:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321133.308729:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6d80 x1853163985601920/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321133.308792:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321133.308796:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321133.308798:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321133.308801:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6376 00000100:00080000:3.0:1767321133.308803:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321133.308806:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321133.308818:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985602176/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321133.308852:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321133.308874:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985602048:0@lo:400:kworker.0 00000100:00100000:3.0:1767321133.308942:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321133.309621:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321133.309635:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985601792:0@lo:400:kworker.0 00000100:00100000:2.0:1767321133.309673:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321133.309990:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985601920:0@lo:38:kworker.0 00000100:00100000:0.0:1767321133.310020:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321133.310055:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985602176:0@lo:38:kworker.0 00000100:00100000:0.0:1767321133.310076:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321133.310118:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602048 00000100:00100000:0.0:1767321133.310128:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602048 00000100:00080000:0.0:1767321133.310155:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321133.310222:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601792 00000100:00100000:0.0:1767321133.310225:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601792 00000100:00080000:0.0:1767321133.310239:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321133.310253:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985601920 00000100:00100000:0.0:1767321133.310256:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985601920 00000100:00100000:0.0:1767321133.310265:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602176 00000100:00100000:0.0:1767321133.310267:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602176 00000100:00100000:0.0:1767321133.310276:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5637 00000100:00100000:0.0:1767321133.310284:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985602048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321133.310334:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985602048:12345-0@lo:400:kworker.0 Request processed in 50us (1390us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321133.310340:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5637 00000100:00100000:0.0:1767321133.310356:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5638 00000100:00100000:0.0:1767321133.310363:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985601792:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321133.310392:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985601792:12345-0@lo:400:kworker.0 Request processed in 29us (718us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321133.310397:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5638 00000100:00100000:0.0:1767321133.310427:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6455 00000100:00100000:0.0:1767321133.310431:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985601920:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321133.310448:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321133.310486:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985601920:12345-0@lo:38:kworker.0 Request processed in 53us (466us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321133.310493:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6455 00000100:00100000:0.0:1767321133.310507:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6456 00000100:00100000:0.0:1767321133.310511:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985602176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321133.310520:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321133.310547:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985602176:12345-0@lo:38:kworker.0 Request processed in 35us (470us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321133.310556:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6456 00000100:00100000:2.0:1767321133.310556:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985601792:0@lo:400:kworker.0 00000100:00080000:0.0:1767321133.310590:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321133.310596:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321133.310601:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985601920:0@lo:38:kworker.0 00000100:00080000:0.0:1767321133.310609:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321133.310617:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321133.310620:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985602176:0@lo:38:kworker.0 00000100:00100000:0.0:1767321133.316477:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985602048:0@lo:400:kworker.0 00010000:00010000:2.0:1767321133.915756:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321134.940374:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321135.964961:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321136.995592:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321137.883845:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321137.883882:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985602304:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321137.884002:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321137.884279:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602304 00000100:00100000:1.0:1767321137.884300:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602304 00000100:00100000:1.0:1767321137.884401:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2868 00000100:00100000:1.0:1767321137.884421:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985602304:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321137.884469:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321137.884533:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985602304:12345-0@lo:13:osp-pre-1-1.0 Request processed in 122us (530us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321137.884541:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2868 00000100:00080000:2.0:1767321137.884777:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321137.884827:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985602304:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321137.885835:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985602432:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321137.885905:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321137.886251:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602432 00000100:00100000:3.0:1767321137.886260:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602432 00000100:00100000:3.0:1767321137.886384:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2869 00000100:00100000:3.0:1767321137.886395:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985602432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321137.886448:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321137.886807:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985602432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 412us (899us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321137.886817:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2869 00000100:00100000:2.0:1767321137.886917:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985602432:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321137.886967:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321138.011598:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321138.347315:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321138.347479:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920656384 00000100:00100000:2.0:1767321138.347493:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920656384 00000100:00100000:2.0:1767321138.347605:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5639 00000100:00100000:2.0:1767321138.347633:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920656384:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321138.347733:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920656384:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 106us (434us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321138.347752:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5639 00000100:00080000:0.0:1767321138.348096:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321138.351526:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321138.352034:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920656256 00000100:00100000:3.0:1767321138.352050:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920656256 00000100:00100000:3.0:1767321138.352119:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6457 00000100:00100000:3.0:1767321138.352130:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920656256:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321138.352161:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321138.352305:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920656256:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 175us (789us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321138.352314:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6457 00000100:00080000:3.0:1767321138.395845:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321138.395860:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321138.395864:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321138.395868:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321138.395875:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321138.395888:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6381 00000100:00080000:3.0:1767321138.395895:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321138.395905:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321138.395954:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4700 x1853163985602560/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321138.395986:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321138.395990:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321138.395992:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321138.395996:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6381 00000100:00080000:3.0:1767321138.395998:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321138.396004:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321138.396021:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7b80 x1853163985602688/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321138.396118:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985602816:0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.396223:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321138.396367:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985602560:0@lo:38:kworker.0 00000100:00100000:0.0:1767321138.396392:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321138.396414:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985602688:0@lo:38:kworker.0 00000100:00100000:0.0:1767321138.396432:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321138.396481:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602688 00000100:00100000:0.0:1767321138.396490:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602688 00000100:00100000:0.0:1767321138.396543:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2105 00000100:00100000:0.0:1767321138.396555:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985602688:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321138.396583:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767321138.397148:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321138.397154:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321138.397156:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321138.397176:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6381 00000100:00080000:3.0:1767321138.397179:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321138.397183:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321138.397208:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca6d80 x1853163985602944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321138.397234:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321138.397236:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321138.397238:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321138.397241:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6381 00000100:00080000:3.0:1767321138.397242:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321138.397245:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321138.397255:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2300 x1853163985603200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321138.397283:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321138.397296:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985603072:0@lo:400:kworker.0 00000100:00100000:2.0:1767321138.397410:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602816 00000100:00100000:2.0:1767321138.397417:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602816 00000100:00100000:2.0:1767321138.397486:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602560 00000100:00100000:2.0:1767321138.397490:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602560 00000100:00100000:2.0:1767321138.397511:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5640 00000100:00100000:2.0:1767321138.397518:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985602816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321138.398075:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321138.398140:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985603072 00000100:00100000:3.0:1767321138.398146:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985603072 00000100:00100000:3.0:1767321138.398214:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5641 00000100:00100000:3.0:1767321138.398222:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985603072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.398432:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6458 00000100:00100000:1.0:1767321138.398439:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985602560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321138.398472:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321138.398474:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985603072:12345-0@lo:400:kworker.0 Request processed in 253us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321138.398482:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5641 00000100:00100000:1.0:1767321138.398512:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985602560:12345-0@lo:38:kworker.0 Request processed in 72us (2119us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321138.398518:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6458 00000100:00100000:3.0:1767321138.398538:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985603072:0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.398547:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985602944:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.398578:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321138.398578:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321138.398590:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985602944 00000100:00100000:3.0:1767321138.398594:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985602944 00000100:00100000:1.0:1767321138.398595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985603200:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.398629:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321138.398656:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321138.398663:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321138.398666:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985602560:0@lo:38:kworker.0 00000100:00100000:0.0:1767321138.398825:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985602688:12345-0@lo:38:kworker.0 Request processed in 2268us (2388us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321138.398835:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2105 00000100:00080000:0.0:1767321138.398871:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321138.398875:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321138.398878:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985602688:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.399086:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985603200 00000100:00100000:1.0:1767321138.399090:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985603200 00000100:00100000:1.0:1767321138.399114:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6459 00000100:00100000:1.0:1767321138.399119:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985602944:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321138.399128:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6460 00010000:00020000:1.0:1767321138.399131:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321138.399133:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985603200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321138.399142:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321138.399184:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985602944:12345-0@lo:38:kworker.0 Request processed in 64us (605us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321138.399191:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6459 00000100:00080000:1.0:1767321138.399222:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321138.399226:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321138.399229:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985602944:0@lo:38:kworker.0 00000100:00080000:1.0:1767321138.399237:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321138.399240:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321138.399243:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985603200:0@lo:38:kworker.0 00000100:00100000:0.0:1767321138.399516:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985603200:12345-0@lo:38:kworker.0 Request processed in 380us (885us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321138.399523:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6460 00000100:00100000:2.0:1767321138.400540:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985602816:12345-0@lo:400:kworker.0 Request processed in 2938us (4232us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321138.400552:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5640 00000100:00100000:1.0:1767321138.400587:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985602816:0@lo:400:kworker.0 00010000:00010000:2.0:1767321139.035988:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321140.060846:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321141.083716:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321142.108969:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321142.442278:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767321142.442314:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920656512 offset 0 length 520: 4 00000100:00100000:1.0:1767321142.442387:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321142.442693:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920656768 00000100:00100000:2.0:1767321142.442713:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920656768 00000100:00100000:2.0:1767321142.442844:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920656640 00000100:00100000:2.0:1767321142.442849:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920656640 00000100:00100000:0.0:1767321142.442867:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5642 00000100:00100000:0.0:1767321142.442887:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920656768:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767321142.442976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321142.443076:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920656768:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 195us (809us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321142.448841:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6461 00000100:00100000:2.0:1767321142.448860:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920656640:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321142.448894:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321142.449018:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920656640:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 158us (6629us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321142.449027:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6461 00000100:00100000:0.0:1767321142.454695:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5642 00000100:00100000:1.0:1767321142.491912:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321117/real 1767321117] req@ffff8ce1a593a680 x1853163985598592/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321142 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321142.491952:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a593a680 x1853163985598592/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321142 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321142.491977:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321142.492011:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321142.492016:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985598592:0@lo:250:kworker.0 00000100:00080000:3.0:1767321142.492570:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321142.492705:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321142.492710:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321142.492717:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6366 00000100:00080000:3.0:1767321142.492723:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321142.492731:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321142.492774:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3100 x1853163985603328/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321142.492837:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985603328:0@lo:250:kworker.0 00000400:00000100:1.0:1767321142.492897:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985603328 offset 0 length 520: 4 00000100:00080000:3.0:1767321142.492967:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321142.492972:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321142.492974:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321142.492978:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6386 00000100:00080000:3.0:1767321142.492981:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321142.492985:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321142.493006:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2300 x1853163985603456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321142.493046:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985603456:0@lo:38:kworker.0 00000100:00100000:1.0:1767321142.493082:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321142.493327:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985603456 00000100:00100000:0.0:1767321142.493338:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985603456 00000100:00100000:0.0:1767321142.493380:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6462 00000100:00100000:0.0:1767321142.493389:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985603456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321142.493415:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321142.493478:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985603456:12345-0@lo:38:kworker.0 Request processed in 89us (395us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321142.493485:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6462 00000100:00080000:3.0:1767321142.493768:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321142.493773:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321142.493776:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321142.493781:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6386 00000100:00080000:3.0:1767321142.493787:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321142.493792:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321142.493816:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4a80 x1853163985603584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321142.493881:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767321142.493928:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321142.493932:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321142.493934:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321142.493938:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6386 00000100:00080000:3.0:1767321142.493940:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321142.493944:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321142.493957:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7b80 x1853163985603840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321142.493990:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321142.493994:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321142.493996:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321142.493999:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6386 00000100:00080000:3.0:1767321142.494001:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321142.494005:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321142.494016:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca7480 x1853163985604096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321142.494091:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985603968:0@lo:400:kworker.0 00000100:00100000:0.0:1767321142.494128:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321142.494154:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321142.494159:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985604224:0@lo:400:kworker.0 00000100:00100000:0.0:1767321142.494207:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321142.494267:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985603968 00000100:00100000:0.0:1767321142.494272:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985603968 00000100:00100000:0.0:1767321142.494330:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5643 00000100:00100000:0.0:1767321142.494337:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985603968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321142.494387:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985603968:12345-0@lo:400:kworker.0 Request processed in 50us (258us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321142.494395:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5643 00000100:00100000:0.0:1767321142.494438:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985603968:0@lo:400:kworker.0 00000100:00100000:0.0:1767321142.494478:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604224 00000100:00100000:0.0:1767321142.494481:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604224 00000100:00100000:0.0:1767321142.494511:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3643 00000100:00100000:0.0:1767321142.494517:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985604224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321142.494553:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985604224:12345-0@lo:400:kworker.0 Request processed in 36us (345us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321142.494560:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3643 00000100:00100000:0.0:1767321142.494597:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985604224:0@lo:400:kworker.0 00000100:00100000:1.0:1767321142.495980:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985604352:0@lo:400:kworker.0 00000100:00100000:1.0:1767321142.496488:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321142.496588:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985603712:0@lo:400:kworker.0 00000100:00100000:1.0:1767321142.496627:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321142.496675:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985603712 00000100:00100000:1.0:1767321142.496684:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985603712 00000100:00100000:1.0:1767321142.496745:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5644 00000100:00100000:1.0:1767321142.496755:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985603712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321142.496809:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985603712:12345-0@lo:400:kworker.0 Request processed in 54us (188us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321142.496817:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5644 00000100:00100000:1.0:1767321142.496867:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985603712:0@lo:400:kworker.0 00000100:00100000:2.0:1767321142.497006:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985603584:0@lo:38:kworker.0 00000100:00100000:2.0:1767321142.497050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321142.497076:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985603840:0@lo:38:kworker.0 00000100:00080000:3.0:1767321142.497095:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321142.497104:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321142.497105:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321142.497130:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985604096:0@lo:38:kworker.0 00000100:00100000:2.0:1767321142.497153:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321142.497208:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985603840 00000100:00080000:2.0:1767321142.497210:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767321142.497212:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985603840 00000100:00080000:2.0:1767321142.497218:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321142.497225:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985603456:0@lo:38:kworker.0 00000100:00100000:3.0:1767321142.497233:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604096 00000100:00100000:3.0:1767321142.497237:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604096 00000100:00100000:3.0:1767321142.497267:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6463 00000100:00100000:3.0:1767321142.497274:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985603840:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321142.497276:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6464 00000100:00100000:2.0:1767321142.497285:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985604096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321142.497306:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321142.497347:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985603840:12345-0@lo:38:kworker.0 Request processed in 74us (243us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321142.497353:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6463 00000100:00080000:3.0:1767321142.497402:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321142.497407:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321142.497410:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985603840:0@lo:38:kworker.0 00000100:00100000:0.0:1767321142.497585:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604352 00000100:00100000:0.0:1767321142.497591:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604352 00000100:00100000:0.0:1767321142.497731:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3644 00000100:00100000:0.0:1767321142.497739:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985604352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321142.497787:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985604352:12345-0@lo:400:kworker.0 Request processed in 49us (1301us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321142.497793:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3644 00000100:00100000:0.0:1767321142.497828:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985603584 00000100:00100000:0.0:1767321142.497831:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985603584 00000100:00100000:0.0:1767321142.497851:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2106 00000100:00100000:0.0:1767321142.497856:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985603584:12345-0@lo:38:kworker.0 00000100:00080000:3.0:1767321142.500737:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321142.501126:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985604352:0@lo:400:kworker.0 00010000:00020000:2.0:1767321142.502074:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321142.502195:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985604096:12345-0@lo:38:kworker.0 Request processed in 4907us (5038us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321142.502209:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6464 00000100:00080000:3.0:1767321142.502299:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321142.502307:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321142.502313:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985604096:0@lo:38:kworker.0 00010000:00020000:0.0:1767321142.503463:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321142.503564:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985603584:12345-0@lo:38:kworker.0 Request processed in 5704us (6511us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321142.503578:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2106 00000100:00080000:3.0:1767321142.503735:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321142.503741:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321142.503745:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985603584:0@lo:38:kworker.0 00000100:00100000:2.0:1767321143.005377:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321143.005387:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321143.005419:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985604608:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321143.005530:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321143.006502:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604608 00000100:00100000:1.0:1767321143.006516:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604608 00000100:00100000:1.0:1767321143.006601:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2870 00000100:00100000:1.0:1767321143.006757:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985604608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321143.006804:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321143.006901:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985604608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 289us (1370us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321143.006910:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2870 00000100:00100000:0.0:1767321143.007032:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985604608:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321143.008792:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985604480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321143.008883:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321143.009874:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604480 00000100:00100000:1.0:1767321143.009892:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604480 00000100:00100000:1.0:1767321143.010044:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2871 00000100:00100000:1.0:1767321143.010058:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985604480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321143.010103:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321143.010256:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985604480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 199us (1371us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321143.010265:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2871 00000100:00100000:3.0:1767321143.010344:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985604480:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321143.131541:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321144.161136:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321145.183100:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321146.026723:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321146.027699:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920656896 00000100:00100000:2.0:1767321146.027712:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920656896 00000100:00100000:2.0:1767321146.027799:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6465 00000100:00080000:3.0:1767321146.028102:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321146.032447:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920656896:12345-192.168.204.26@tcp:17:kworker.0 00000100:00100000:2.0:1767321146.033497:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920656896:12345-192.168.204.26@tcp:17:kworker.0 Request processed in 5682us (6778us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321146.033513:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6465 00010000:00010000:2.0:1767321146.203647:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321147.227654:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321147.567341:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321147.567385:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321147.567571:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657024 00000100:00100000:0.0:1767321147.567710:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657152 00000100:00100000:0.0:1767321147.567726:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657152 00000100:00100000:2.0:1767321147.567727:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657024 00000100:00100000:2.0:1767321147.567925:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5645 00000100:00100000:2.0:1767321147.567940:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920657152:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321147.568062:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920657152:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 126us (674us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321147.568071:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5645 00000100:00100000:2.0:1767321147.568097:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6466 00000100:00100000:2.0:1767321147.568102:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920657024:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321147.568126:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321147.600425:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920657024:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 32309us (33088us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321147.600441:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6466 00000100:00080000:2.0:1767321147.611467:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321147.611477:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321147.611480:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321147.611484:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321147.611488:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321147.611496:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6390 00000100:00080000:2.0:1767321147.611501:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321147.611509:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321147.611549:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2300 x1853163985604736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321147.611581:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321147.611584:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321147.611586:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321147.611589:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6390 00000100:00080000:2.0:1767321147.611592:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321147.611595:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321147.611610:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985604864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321147.611645:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321147.611649:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321147.611651:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321147.611656:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6390 00000100:00080000:2.0:1767321147.611659:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321147.611664:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321147.611665:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985604736:0@lo:38:kworker.0 00000100:00100000:2.0:1767321147.611677:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21500 x1853163985605120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321147.611707:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321147.611711:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321147.611712:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321147.611716:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6390 00000100:00080000:2.0:1767321147.611719:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321147.611723:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321147.611735:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22a00 x1853163985605376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321147.611745:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321147.611786:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604736 00000100:00100000:2.0:1767321147.611793:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604736 00000100:00100000:3.0:1767321147.611814:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985604864:0@lo:38:kworker.0 00000100:00100000:2.0:1767321147.611836:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6467 00000100:00100000:2.0:1767321147.611848:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985604736:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321147.611849:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767321147.611872:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321147.611884:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985605120:0@lo:38:kworker.0 00000100:00100000:3.0:1767321147.611919:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321147.611926:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985604736:12345-0@lo:38:kworker.0 Request processed in 80us (183us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321147.611935:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6467 00000100:00100000:3.0:1767321147.611944:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985605376:0@lo:38:kworker.0 00000100:00100000:2.0:1767321147.611952:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985605120 00000100:00100000:2.0:1767321147.611956:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985605120 00000100:00100000:3.0:1767321147.611970:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321147.612006:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321147.612012:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321147.612019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985604736:0@lo:38:kworker.0 00000100:00100000:1.0:1767321147.612020:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985604992:0@lo:400:kworker.0 00000100:00100000:1.0:1767321147.612056:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321147.612061:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985605248:0@lo:400:kworker.0 00000100:00100000:3.0:1767321147.612096:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985605504:0@lo:400:kworker.0 00000100:00100000:3.0:1767321147.612123:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321147.612203:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985605376 00000100:00100000:2.0:1767321147.612208:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985605376 00000100:00100000:2.0:1767321147.612262:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6468 00000100:00100000:2.0:1767321147.612267:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985605120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321147.612280:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321147.612309:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985605120:12345-0@lo:38:kworker.0 Request processed in 41us (390us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321147.612316:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6468 00000100:00100000:2.0:1767321147.612327:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6469 00000100:00100000:2.0:1767321147.612331:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985605376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321147.612340:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321147.612359:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985605376:12345-0@lo:38:kworker.0 Request processed in 26us (389us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321147.612364:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6469 00000100:00080000:2.0:1767321147.612410:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321147.612414:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321147.612418:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985605120:0@lo:38:kworker.0 00000100:00080000:2.0:1767321147.612425:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321147.612429:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321147.612431:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985605376:0@lo:38:kworker.0 00000100:00100000:2.0:1767321147.612474:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604864 00000100:00100000:2.0:1767321147.612479:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604864 00000100:00100000:2.0:1767321147.612512:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2107 00000100:00100000:2.0:1767321147.612516:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985604864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321147.612526:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321147.612557:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985604864:12345-0@lo:38:kworker.0 Request processed in 39us (712us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321147.612563:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2107 00000100:00100000:2.0:1767321147.612590:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985605632:0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.612613:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321147.612644:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321147.612648:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321147.612651:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985604864:0@lo:38:kworker.0 00000100:00100000:2.0:1767321147.612675:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985605504 00000100:00100000:2.0:1767321147.612678:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985605504 00000100:00100000:2.0:1767321147.612712:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985605632 00000100:00100000:2.0:1767321147.612715:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985605632 00000100:00100000:2.0:1767321147.612732:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3645 00000100:00100000:2.0:1767321147.612739:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985605504:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.612778:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985605504:12345-0@lo:400:kworker.0 Request processed in 40us (653us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321147.612784:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3645 00000100:00100000:2.0:1767321147.612796:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3646 00000100:00100000:2.0:1767321147.612801:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985605632:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321147.613063:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321147.613068:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321147.613106:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985605632:12345-0@lo:400:kworker.0 Request processed in 26us (213us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321147.613113:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3646 00000100:00100000:2.0:1767321147.613142:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985605632:0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.613210:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985605504:0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.615547:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321147.615656:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985604992 00000100:00100000:3.0:1767321147.615666:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985604992 00000100:00100000:3.0:1767321147.615741:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5646 00000100:00100000:3.0:1767321147.615752:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985604992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321147.615823:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985604992:12345-0@lo:400:kworker.0 Request processed in 75us (3767us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321147.615831:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5646 00000100:00100000:1.0:1767321147.615923:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985604992:0@lo:400:kworker.0 00000100:00080000:0.0:1767321147.618356:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321147.619848:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985605248 00000100:00100000:3.0:1767321147.619862:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985605248 00000100:00080000:3.0:1767321147.619906:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321147.619957:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5647 00000100:00100000:2.0:1767321147.619968:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985605248:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.620041:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985605248:12345-0@lo:400:kworker.0 Request processed in 73us (4499us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321147.620053:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5647 00000100:00100000:0.0:1767321147.620269:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985605248:0@lo:400:kworker.0 00000100:00100000:1.0:1767321148.123503:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985605760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321148.123581:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321148.124026:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985605760 00000100:00100000:3.0:1767321148.124034:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985605760 00000100:00100000:3.0:1767321148.124084:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2872 00000100:00100000:3.0:1767321148.124094:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985605760:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321148.124127:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321148.124212:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985605760:12345-0@lo:13:osp-pre-0-1.0 Request processed in 120us (630us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321148.124220:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2872 00000100:00100000:2.0:1767321148.124310:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985605760:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321148.124503:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321148.124514:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985605888:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321148.124543:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321148.124689:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985605888 00000100:00100000:0.0:1767321148.124695:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985605888 00000100:00100000:0.0:1767321148.124731:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2873 00000100:00100000:0.0:1767321148.124739:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985605888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321148.124755:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321148.124787:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985605888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (243us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321148.124794:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2873 00000100:00100000:2.0:1767321148.124864:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985605888:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321148.251414:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321149.275480:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321150.299893:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321151.323535:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321152.350905:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321152.691601:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321152.691652:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321152.691766:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657280 00000100:00100000:0.0:1767321152.691777:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657280 00000100:00100000:0.0:1767321152.691819:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657408 00000100:00100000:0.0:1767321152.691822:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657408 00000100:00100000:0.0:1767321152.691865:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5648 00000100:00100000:0.0:1767321152.691875:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920657408:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321152.691951:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920657408:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (298us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321152.691959:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5648 00000100:00100000:0.0:1767321152.691980:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6470 00000100:00100000:0.0:1767321152.691984:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920657280:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321152.692001:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321152.692029:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920657280:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (439us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321152.692035:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6470 00000100:00080000:3.0:1767321152.692154:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321152.731378:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321152.731387:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321152.731390:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321152.731394:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321152.731398:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321152.731406:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6395 00000100:00080000:2.0:1767321152.731411:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321152.731417:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321152.731457:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985606016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321152.732392:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321152.732397:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321152.732399:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321152.732404:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6395 00000100:00080000:2.0:1767321152.732407:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321152.732412:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321152.732434:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21500 x1853163985606144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321152.732539:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985606272:0@lo:400:kworker.0 00000100:00100000:0.0:1767321152.732606:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321152.732708:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606272 00000100:00100000:1.0:1767321152.732717:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606272 00000100:00100000:1.0:1767321152.732774:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5649 00000100:00100000:1.0:1767321152.732784:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985606272:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321152.732851:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321152.732855:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321152.732858:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321152.732862:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6395 00000100:00080000:2.0:1767321152.732864:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321152.732868:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321152.732882:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe23800 x1853163985606400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321152.732917:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321152.732921:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321152.732923:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321152.732926:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6395 00000100:00080000:2.0:1767321152.732929:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321152.732932:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321152.732943:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21f80 x1853163985606656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321152.733014:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321152.733031:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985606016:0@lo:38:kworker.0 00000100:00100000:3.0:1767321152.733071:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321152.733095:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985606144:0@lo:38:kworker.0 00000100:00100000:3.0:1767321152.733119:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321152.733142:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985606400:0@lo:38:kworker.0 00000100:00100000:3.0:1767321152.733161:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321152.733205:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985606656:0@lo:38:kworker.0 00000100:00100000:3.0:1767321152.733225:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321152.733254:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606016 00000100:00100000:3.0:1767321152.733258:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606016 00000100:00100000:3.0:1767321152.733277:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606400 00000100:00100000:3.0:1767321152.733279:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606400 00000100:00100000:3.0:1767321152.733287:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606656 00000100:00100000:3.0:1767321152.733289:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606656 00000100:00100000:3.0:1767321152.733298:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6471 00000100:00100000:3.0:1767321152.733303:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985606016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321152.733322:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321152.733356:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985606016:12345-0@lo:38:kworker.0 Request processed in 50us (283us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321152.733363:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6471 00000100:00100000:3.0:1767321152.733380:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6472 00000100:00100000:3.0:1767321152.733384:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985606400:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321152.733394:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321152.733417:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985606400:12345-0@lo:38:kworker.0 Request processed in 30us (254us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321152.733422:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6472 00000100:00100000:3.0:1767321152.733433:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6473 00000100:00100000:3.0:1767321152.733437:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985606656:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321152.733446:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321152.733481:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985606656:12345-0@lo:38:kworker.0 Request processed in 42us (254us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321152.733487:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6473 00000100:00080000:3.0:1767321152.733518:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321152.733523:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321152.733527:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985606016:0@lo:38:kworker.0 00000100:00080000:3.0:1767321152.733536:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321152.733539:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321152.733542:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985606400:0@lo:38:kworker.0 00000100:00080000:3.0:1767321152.733548:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321152.733551:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321152.733554:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985606656:0@lo:38:kworker.0 00000100:00100000:3.0:1767321152.733602:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985606784:0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.733630:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321152.733653:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321152.733658:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985606528:0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.733678:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321152.733700:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985606912:0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.733721:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321152.733755:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606528 00000100:00100000:3.0:1767321152.733759:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606528 00000100:00080000:3.0:1767321152.733776:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321152.733799:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5650 00000100:00100000:3.0:1767321152.733806:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985606528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.733839:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985606528:12345-0@lo:400:kworker.0 Request processed in 34us (161us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321152.733850:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5650 00000100:00100000:3.0:1767321152.733876:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985606528:0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.733918:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606784 00000100:00100000:3.0:1767321152.733922:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606784 00000100:00100000:3.0:1767321152.733954:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606912 00000100:00100000:3.0:1767321152.733958:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606912 00000100:00100000:3.0:1767321152.733979:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3647 00000100:00100000:3.0:1767321152.733986:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985606784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.734020:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985606784:12345-0@lo:400:kworker.0 Request processed in 34us (388us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321152.734026:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3647 00000100:00100000:3.0:1767321152.734036:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3648 00000100:00100000:3.0:1767321152.734042:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985606912:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.734086:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985606912:12345-0@lo:400:kworker.0 Request processed in 43us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321152.734091:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3648 00000100:00080000:3.0:1767321152.734106:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321152.734108:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321152.734124:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985606912:0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.734145:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985606784:0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.734204:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985606144 00000100:00100000:3.0:1767321152.734207:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985606144 00000100:00100000:3.0:1767321152.734228:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2108 00000100:00100000:3.0:1767321152.734232:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985606144:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321152.734243:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321152.734275:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985606144:12345-0@lo:38:kworker.0 Request processed in 41us (1155us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321152.734281:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2108 00000100:00080000:3.0:1767321152.734305:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321152.734308:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321152.734311:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985606144:0@lo:38:kworker.0 00000100:00100000:1.0:1767321152.743529:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985606272:12345-0@lo:400:kworker.0 Request processed in 10734us (10909us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321152.743543:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5649 00000100:00100000:0.0:1767321152.746867:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985606272:0@lo:400:kworker.0 00000100:00100000:3.0:1767321153.246077:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321153.246103:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985607040:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321153.246211:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321153.246338:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321153.246350:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985607168:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321153.246380:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321153.246452:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607040 00000100:00100000:1.0:1767321153.246460:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607040 00000100:00100000:1.0:1767321153.246510:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607168 00000100:00100000:1.0:1767321153.246514:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607168 00000100:00100000:1.0:1767321153.246536:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2874 00000100:00100000:1.0:1767321153.246545:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985607040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321153.246580:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321153.246627:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985607040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (417us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321153.246641:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2874 00000100:00100000:1.0:1767321153.246657:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2875 00000100:00100000:1.0:1767321153.246663:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985607168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321153.246677:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321153.246703:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985607168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 41us (323us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321153.246709:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2875 00000100:00100000:3.0:1767321153.246966:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985607040:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321153.251239:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a593aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985607168:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321153.371399:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321154.398393:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321155.420072:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321155.803381:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321155.803404:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321155.803408:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321155.803413:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321155.803418:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321155.803428:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6400 00000100:00080000:2.0:1767321155.803434:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321155.803441:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321155.803484:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20e00 x1853163985607296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321155.803516:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321155.803520:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321155.803521:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321155.803525:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6400 00000100:00080000:2.0:1767321155.803527:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321155.803531:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321155.803547:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21180 x1853163985607424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321155.803593:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985607296:0@lo:38:kworker.0 00000100:00080000:2.0:1767321155.803602:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321155.803608:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321155.803610:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321155.803615:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6400 00000100:00080000:2.0:1767321155.803617:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321155.803621:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321155.803637:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20700 x1853163985607680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321155.803672:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321155.803676:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321155.803678:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321155.803683:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6400 00000100:00080000:2.0:1767321155.803686:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321155.803690:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321155.803698:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321155.803703:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21880 x1853163985607936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321155.803744:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985608064:0@lo:400:kworker.0 00000100:00100000:3.0:1767321155.803778:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985607424:0@lo:38:kworker.0 00000100:00100000:2.0:1767321155.803792:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321155.803818:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321155.803826:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321155.803835:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985607808:0@lo:400:kworker.0 00000100:00100000:2.0:1767321155.803867:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321155.803877:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985607680:0@lo:38:kworker.0 00000100:00100000:3.0:1767321155.803910:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321155.803913:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608064 00000100:00100000:2.0:1767321155.803927:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608064 00000100:00100000:3.0:1767321155.803933:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985607936:0@lo:38:kworker.0 00000100:00100000:3.0:1767321155.803965:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321155.804028:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3649 00000100:00100000:3.0:1767321155.804038:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607424 00000100:00100000:2.0:1767321155.804041:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985608064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321155.804044:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607424 00000100:00100000:3.0:1767321155.804079:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2109 00000100:00100000:3.0:1767321155.804086:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985607424:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321155.804111:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985608064:12345-0@lo:400:kworker.0 Request processed in 71us (318us total) trans 0 rc 0/0 00010000:00020000:3.0:1767321155.804111:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321155.804123:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3649 00000100:00100000:2.0:1767321155.804156:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607296 00000100:00100000:3.0:1767321155.804188:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985607424:12345-0@lo:38:kworker.0 Request processed in 73us (342us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321155.804189:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607296 00000100:00100000:3.0:1767321155.804199:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2109 00000100:00100000:2.0:1767321155.804217:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607808 00000100:00100000:2.0:1767321155.804222:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607808 00000100:00100000:3.0:1767321155.804226:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607680 00000100:00100000:3.0:1767321155.804232:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607680 00000100:00100000:2.0:1767321155.804249:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607936 00000100:00100000:2.0:1767321155.804254:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607936 00000100:00100000:3.0:1767321155.804257:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5651 00000100:00100000:3.0:1767321155.804267:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985607808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321155.804274:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6474 00000100:00100000:2.0:1767321155.804281:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985607296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321155.804293:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321155.804303:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985607808:12345-0@lo:400:kworker.0 Request processed in 36us (435us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321155.804310:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5651 00000100:00100000:3.0:1767321155.804327:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6475 00000100:00100000:2.0:1767321155.804333:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985607296:12345-0@lo:38:kworker.0 Request processed in 48us (635us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321155.804337:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985607680:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321155.804346:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6474 00010000:00020000:3.0:1767321155.804353:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321155.804364:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6476 00000100:00100000:2.0:1767321155.804370:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985607936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321155.804382:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321155.804385:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985607680:12345-0@lo:38:kworker.0 Request processed in 47us (473us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321155.804393:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6475 00000100:00100000:2.0:1767321155.804417:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985607936:12345-0@lo:38:kworker.0 Request processed in 45us (448us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321155.804425:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6476 00000100:00080000:2.0:1767321155.804447:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321155.804448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321155.804453:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321155.804456:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321155.804462:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985607680:0@lo:38:kworker.0 00000100:00080000:3.0:1767321155.804475:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767321155.804478:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985608192:0@lo:400:kworker.0 00000100:00080000:3.0:1767321155.804481:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321155.804488:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985607936:0@lo:38:kworker.0 00000100:00080000:3.0:1767321155.804501:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321155.804505:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321155.804509:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985607296:0@lo:38:kworker.0 00000100:00100000:2.0:1767321155.804516:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321155.804523:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321155.804529:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321155.804532:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985607424:0@lo:38:kworker.0 00000100:00100000:2.0:1767321155.804546:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321155.804556:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985607552:0@lo:400:kworker.0 00000100:00100000:2.0:1767321155.804582:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321155.804601:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985607808:0@lo:400:kworker.0 00000100:00100000:3.0:1767321155.804617:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985608064:0@lo:400:kworker.0 00000100:00100000:2.0:1767321155.804618:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608192 00000100:00100000:2.0:1767321155.804624:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608192 00000100:00100000:3.0:1767321155.804646:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985607552 00000100:00100000:3.0:1767321155.804651:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985607552 00000100:00100000:2.0:1767321155.804660:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3650 00000100:00100000:2.0:1767321155.804669:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985608192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321155.804689:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5652 00000100:00100000:3.0:1767321155.804699:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985607552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321155.804711:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985608192:12345-0@lo:400:kworker.0 Request processed in 43us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321155.804719:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3650 00000100:00080000:2.0:1767321155.804738:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321155.804743:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985607552:12345-0@lo:400:kworker.0 Request processed in 44us (158us total) trans 0 rc 0/0 00000100:00080000:2.0:1767321155.804744:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321155.804753:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5652 00000100:00100000:2.0:1767321155.804765:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985607552:0@lo:400:kworker.0 00000100:00100000:2.0:1767321155.804777:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985608192:0@lo:400:kworker.0 00010000:00010000:1.0:1767321156.443973:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321157.470646:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321157.803288:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321157.803344:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321157.803491:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657664 00000100:00100000:1.0:1767321157.803523:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657664 00000100:00100000:1.0:1767321157.803611:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657536 00000100:00100000:1.0:1767321157.803615:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657536 00000100:00100000:1.0:1767321157.803641:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5653 00000100:00100000:1.0:1767321157.803654:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920657664:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321157.803751:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920657664:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (479us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321157.803759:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5653 00000100:00100000:1.0:1767321157.803783:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6477 00000100:00100000:1.0:1767321157.803789:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920657536:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321157.803811:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321157.803844:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920657536:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (498us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321157.803853:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6477 00000100:00080000:3.0:1767321157.804324:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321158.363809:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321158.363840:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985608320:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321158.363934:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321158.364156:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321158.364197:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985608448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321158.365803:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321158.369399:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608320 00000100:00100000:0.0:1767321158.369421:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608320 00000100:00080000:3.0:1767321158.372880:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321158.372996:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608448 00000100:00100000:0.0:1767321158.373012:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608448 00000100:00080000:3.0:1767321158.373075:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321158.374440:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2876 00000100:00100000:1.0:1767321158.374463:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985608320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321158.374514:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321158.374637:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985608320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 183us (10700us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321158.374647:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2876 00000100:00100000:1.0:1767321158.374674:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2877 00000100:00100000:1.0:1767321158.374682:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985608448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321158.374706:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321158.374745:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985608448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (8952us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321158.374752:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2877 00000100:00100000:3.0:1767321158.374885:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985608448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321158.374941:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985608320:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321158.491456:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321159.515455:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321160.539447:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321160.923347:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321160.923361:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321160.923371:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321160.923376:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321160.923381:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321160.923391:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6404 00000100:00080000:2.0:1767321160.923397:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321160.923407:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321160.923472:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21500 x1853163985608576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321160.923502:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321160.923506:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321160.923510:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321160.923513:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6404 00000100:00080000:2.0:1767321160.923516:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321160.923520:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321160.923534:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20e00 x1853163985608704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321160.923581:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321160.923585:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321160.923587:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321160.923591:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6404 00000100:00080000:2.0:1767321160.923594:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321160.923597:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321160.923613:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20700 x1853163985608960/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321160.923640:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321160.923643:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321160.923645:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321160.923648:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6404 00000100:00080000:2.0:1767321160.923651:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321160.923654:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321160.923669:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe23100 x1853163985609216/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321160.923718:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985609472:0@lo:400:kworker.0 00000100:00100000:2.0:1767321160.923820:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321160.923858:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985608576:0@lo:38:kworker.0 00000100:00100000:2.0:1767321160.923893:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321160.923895:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321160.923915:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985608832:0@lo:400:kworker.0 00000100:00100000:3.0:1767321160.923932:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985608704:0@lo:38:kworker.0 00000100:00100000:2.0:1767321160.923938:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321160.923959:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321160.923964:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985609344:0@lo:400:kworker.0 00000100:00100000:3.0:1767321160.923979:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985608960:0@lo:38:kworker.0 00000100:00100000:2.0:1767321160.923987:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321160.923997:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321160.924006:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321160.924011:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985609088:0@lo:400:kworker.0 00000100:00100000:3.0:1767321160.924012:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985609216:0@lo:38:kworker.0 00000100:00100000:2.0:1767321160.924030:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321160.924062:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609472 00000100:00100000:2.0:1767321160.924072:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609472 00000100:00100000:3.0:1767321160.924116:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608576 00000100:00100000:3.0:1767321160.924121:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608576 00000100:00100000:3.0:1767321160.924185:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608832 00000100:00100000:3.0:1767321160.924189:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608832 00000100:00080000:0.0:1767321160.924194:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321160.924200:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321160.924211:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608960 00000100:00100000:0.0:1767321160.924215:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608960 00000100:00100000:3.0:1767321160.924219:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609088 00000100:00100000:3.0:1767321160.924221:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609088 00000100:00100000:0.0:1767321160.924236:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5654 00000100:00100000:0.0:1767321160.924885:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985608832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321160.925126:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985608832:12345-0@lo:400:kworker.0 Request processed in 879us (1183us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321160.925247:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5655 00000100:00100000:3.0:1767321160.925261:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985609088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321160.925262:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5654 00000100:00100000:0.0:1767321160.925288:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6478 00000100:00100000:0.0:1767321160.925293:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985608576:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321160.925306:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985609088:12345-0@lo:400:kworker.0 Request processed in 52us (1275us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321160.925312:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5655 00010000:00020000:0.0:1767321160.925318:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321160.925323:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6479 00000100:00100000:3.0:1767321160.925327:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985608960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321160.925336:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321160.925347:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985608576:12345-0@lo:38:kworker.0 Request processed in 53us (1451us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321160.925355:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6478 00000100:00100000:3.0:1767321160.925355:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985608960:12345-0@lo:38:kworker.0 Request processed in 27us (1357us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321160.925360:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6479 00000100:00080000:3.0:1767321160.925394:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321160.925438:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985609088:0@lo:400:kworker.0 00000100:00100000:3.0:1767321160.925485:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321160.925529:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321160.925535:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321160.925539:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985608576:0@lo:38:kworker.0 00000100:00080000:3.0:1767321160.925550:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321160.925553:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321160.925556:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985608960:0@lo:38:kworker.0 00000100:00100000:3.0:1767321160.925582:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609216 00000100:00100000:3.0:1767321160.925587:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609216 00000100:00100000:3.0:1767321160.925613:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6480 00000100:00100000:3.0:1767321160.925618:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985609216:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321160.925628:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321160.925632:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985608704 00000100:00100000:0.0:1767321160.925637:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985608704 00000100:00100000:0.0:1767321160.925661:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2110 00000100:00100000:3.0:1767321160.925661:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985609216:12345-0@lo:38:kworker.0 Request processed in 41us (175us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321160.925666:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985608704:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321160.925667:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6480 00010000:00020000:0.0:1767321160.925676:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767321160.925696:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321160.925701:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321160.925703:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985608704:12345-0@lo:38:kworker.0 Request processed in 37us (1745us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321160.925704:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985609216:0@lo:38:kworker.0 00000100:00100000:0.0:1767321160.925714:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2110 00000100:00080000:3.0:1767321160.925719:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321160.925722:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321160.925725:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985608704:0@lo:38:kworker.0 00000100:00100000:3.0:1767321160.925764:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609344 00000100:00100000:3.0:1767321160.925767:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609344 00000100:00100000:3.0:1767321160.925807:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3651 00000100:00100000:3.0:1767321160.925812:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985609344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321160.925851:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985609344:12345-0@lo:400:kworker.0 Request processed in 38us (1862us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321160.925858:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3651 00000100:00080000:3.0:1767321160.925877:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321160.926594:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3652 00000100:00100000:3.0:1767321160.926594:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985609344:0@lo:400:kworker.0 00000100:00100000:2.0:1767321160.926608:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985609472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321160.926642:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985608832:0@lo:400:kworker.0 00000100:00100000:2.0:1767321160.926659:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985609472:12345-0@lo:400:kworker.0 Request processed in 57us (2840us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321160.926659:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985609472:0@lo:400:kworker.0 00000100:00100000:2.0:1767321160.926665:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3652 00010000:00010000:0.0:1767321161.564969:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321162.589326:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321162.922735:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321162.922858:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657920 00000100:00100000:2.0:1767321162.922902:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657920 00000100:00100000:2.0:1767321162.923005:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5656 00000100:00100000:2.0:1767321162.923017:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920657920:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321162.923131:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920657920:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 116us (406us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321162.923141:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5656 00000100:00080000:3.0:1767321162.923628:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321162.924768:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321162.925736:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920657792 00000100:00100000:0.0:1767321162.925751:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920657792 00000100:00100000:0.0:1767321162.925877:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6481 00000100:00100000:0.0:1767321162.925888:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920657792:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321162.925920:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321162.926011:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920657792:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 122us (1252us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321162.926019:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6481 00000100:00100000:3.0:1767321163.483740:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321163.483762:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985609600:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321163.483832:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321163.484020:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609600 00000100:00100000:1.0:1767321163.484029:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609600 00000100:00100000:1.0:1767321163.484088:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2878 00000100:00100000:1.0:1767321163.484122:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985609600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321163.484155:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321163.484260:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985609600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 162us (426us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321163.484268:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2878 00000100:00100000:0.0:1767321163.484660:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985609600:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321163.485161:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985609728:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321163.485237:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321163.485289:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321163.490418:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609728 00000100:00100000:2.0:1767321163.490439:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609728 00000100:00100000:2.0:1767321163.490552:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2879 00000100:00100000:2.0:1767321163.490566:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985609728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321163.490611:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321163.490736:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985609728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 174us (5498us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321163.490746:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2879 00000100:00100000:0.0:1767321163.491117:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985609728:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321163.491656:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321163.611536:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321164.635583:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321165.659668:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321166.043806:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321166.043829:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321166.043833:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321166.043838:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321166.043845:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321166.043857:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6409 00000100:00080000:3.0:1767321166.043863:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321166.043871:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321166.043926:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5180 x1853163985609856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321166.043954:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321166.043958:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321166.043960:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321166.043964:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6409 00000100:00080000:3.0:1767321166.043966:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321166.043970:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321166.043983:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21880 x1853163985609984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321166.044087:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985609856:0@lo:38:kworker.0 00000100:00100000:0.0:1767321166.044208:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321166.044295:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985609984:0@lo:38:kworker.0 00000100:00100000:0.0:1767321166.044323:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321166.044478:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609984 00000100:00100000:2.0:1767321166.044488:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609984 00000100:00100000:2.0:1767321166.044541:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2111 00000100:00100000:2.0:1767321166.044549:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985609984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321166.044578:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321166.044627:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985609984:12345-0@lo:38:kworker.0 Request processed in 78us (302us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321166.044634:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2111 00000100:00080000:3.0:1767321166.044723:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321166.044727:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321166.044729:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321166.044734:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6409 00000100:00080000:3.0:1767321166.044737:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321166.044741:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321166.044759:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe23100 x1853163985610240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321166.044790:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321166.044794:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321166.044796:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321166.044799:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6409 00000100:00080000:3.0:1767321166.044802:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321166.044805:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321166.044828:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21500 x1853163985610496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321166.044872:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985610624:0@lo:400:kworker.0 00000100:00100000:3.0:1767321166.044908:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321166.044931:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321166.044936:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985610368:0@lo:400:kworker.0 00000100:00100000:3.0:1767321166.044961:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321166.045038:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985610112:0@lo:400:kworker.0 00000100:00100000:2.0:1767321166.045068:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321166.045092:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321166.045105:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985610752:0@lo:400:kworker.0 00000100:00100000:2.0:1767321166.045125:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321166.045153:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985610624 00000100:00100000:2.0:1767321166.045157:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985610624 00000100:00100000:2.0:1767321166.045219:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985610752 00000100:00100000:2.0:1767321166.045222:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985610752 00000100:00100000:2.0:1767321166.045241:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3653 00000100:00100000:2.0:1767321166.045246:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985610624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321166.045291:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985610624:12345-0@lo:400:kworker.0 Request processed in 45us (382us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321166.045298:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3653 00000100:00100000:2.0:1767321166.045308:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3654 00000100:00100000:2.0:1767321166.045313:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985610752:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321166.045337:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985610752:12345-0@lo:400:kworker.0 Request processed in 25us (212us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321166.045342:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3654 00000100:00100000:2.0:1767321166.045358:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985609856 00000100:00100000:2.0:1767321166.045361:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985609856 00000100:00100000:2.0:1767321166.045376:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985610368 00000100:00100000:2.0:1767321166.045378:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985610368 00000100:00100000:2.0:1767321166.045395:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985610112 00000100:00100000:2.0:1767321166.045397:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985610112 00000100:00100000:2.0:1767321166.045412:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5657 00000100:00100000:2.0:1767321166.045417:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985610368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321166.045442:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985610368:12345-0@lo:400:kworker.0 Request processed in 25us (480us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321166.045447:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5657 00000100:00100000:2.0:1767321166.045457:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5658 00000100:00100000:2.0:1767321166.045463:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985610112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321166.045485:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985610112:12345-0@lo:400:kworker.0 Request processed in 23us (417us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321166.045491:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5658 00000100:00100000:2.0:1767321166.045500:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6482 00000100:00100000:2.0:1767321166.045504:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985609856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321166.045516:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321166.045539:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985609856:12345-0@lo:38:kworker.0 Request processed in 34us (1333us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321166.045545:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6482 00000100:00100000:2.0:1767321166.045586:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985610752:0@lo:400:kworker.0 00000100:00100000:2.0:1767321166.045597:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985610112:0@lo:400:kworker.0 00000100:00100000:0.0:1767321166.046416:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985610240:0@lo:38:kworker.0 00000100:00100000:0.0:1767321166.046484:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321166.046531:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985610496:0@lo:38:kworker.0 00000100:00100000:0.0:1767321166.046555:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321166.046589:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321166.046597:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321166.046601:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985609856:0@lo:38:kworker.0 00000100:00080000:0.0:1767321166.046616:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321166.046619:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321166.046623:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985609984:0@lo:38:kworker.0 00000100:00100000:3.0:1767321166.046738:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985610240 00000100:00100000:3.0:1767321166.046745:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985610240 00000100:00100000:3.0:1767321166.046771:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985610496 00000100:00100000:3.0:1767321166.046773:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985610496 00000100:00100000:3.0:1767321166.046786:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6483 00000100:00100000:3.0:1767321166.046793:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985610240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321166.046820:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321166.046875:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985610240:12345-0@lo:38:kworker.0 Request processed in 80us (390us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321166.046883:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6483 00000100:00100000:3.0:1767321166.046899:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6484 00000100:00100000:3.0:1767321166.046903:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985610496:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321166.046912:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321166.046935:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985610496:12345-0@lo:38:kworker.0 Request processed in 31us (379us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321166.046940:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6484 00000100:00080000:3.0:1767321166.046957:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321166.046963:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321166.046966:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321166.047384:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321166.047389:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321166.047392:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985610240:0@lo:38:kworker.0 00000100:00080000:0.0:1767321166.047400:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321166.047403:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321166.047405:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985610496:0@lo:38:kworker.0 00000100:00100000:3.0:1767321166.047786:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985610368:0@lo:400:kworker.0 00000100:00100000:3.0:1767321166.047805:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985610624:0@lo:400:kworker.0 00010000:00010000:2.0:1767321166.685024:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321167.067390:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321142/real 1767321142] req@ffff8ce1b2dc3100 x1853163985603328/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321167 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321167.067435:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b2dc3100 x1853163985603328/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321167 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767321167.067460:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321167.067494:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767321167.067499:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985603328:0@lo:250:kworker.0 00000100:00080000:3.0:1767321167.067621:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321167.067627:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321167.067632:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321167.067638:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6390 00000100:00080000:3.0:1767321167.067643:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321167.067650:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321167.067704:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffdf80 x1853163985610880/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321167.067771:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985610880:0@lo:250:kworker.0 00000400:00000100:0.0:1767321167.067863:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985610880 offset 0 length 520: 4 00000100:00080000:3.0:1767321167.067921:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321167.067926:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321167.067928:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321167.067933:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6414 00000100:00080000:3.0:1767321167.067936:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321167.067948:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321167.067967:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fffb80 x1853163985611008/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321167.070618:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985611008:0@lo:38:kworker.0 00000100:00100000:0.0:1767321167.072138:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321167.074917:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321167.074931:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321167.074936:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321167.074955:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6414 00000100:00080000:3.0:1767321167.074963:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321167.074973:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321167.075029:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc000 x1853163985611136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321167.075090:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321167.075095:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321167.075097:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321167.075102:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6414 00000100:00080000:3.0:1767321167.075105:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321167.075109:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321167.075124:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffd180 x1853163985611392/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321167.075148:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321167.075151:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321167.075154:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321167.075157:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6414 00000100:00080000:3.0:1767321167.075159:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321167.075181:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321167.075194:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc700 x1853163985611648/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321167.075262:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985611776:0@lo:400:kworker.0 00000100:00100000:3.0:1767321167.075350:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321167.075401:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321167.075407:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985611520:0@lo:400:kworker.0 00000100:00100000:3.0:1767321167.075433:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321167.075514:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985611904:0@lo:400:kworker.0 00000100:00100000:0.0:1767321167.075547:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321167.075575:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321167.075580:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985611264:0@lo:400:kworker.0 00000100:00100000:0.0:1767321167.075611:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985611136:0@lo:38:kworker.0 00000100:00100000:0.0:1767321167.075639:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321167.075658:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985611392:0@lo:38:kworker.0 00000100:00100000:0.0:1767321167.075691:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611136 00000100:00100000:0.0:1767321167.075700:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611136 00000100:00100000:0.0:1767321167.075760:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2112 00000100:00100000:0.0:1767321167.075769:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985611136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321167.075803:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321167.075850:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985611136:12345-0@lo:38:kworker.0 Request processed in 81us (210us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321167.075858:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2112 00000100:00100000:0.0:1767321167.075910:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611776 00000100:00100000:0.0:1767321167.075916:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611776 00000100:00100000:0.0:1767321167.075951:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611904 00000100:00100000:0.0:1767321167.075954:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611904 00000100:00100000:0.0:1767321167.075976:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3655 00000100:00100000:0.0:1767321167.075982:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985611776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321167.076027:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985611776:12345-0@lo:400:kworker.0 Request processed in 45us (678us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321167.076034:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3655 00000100:00100000:0.0:1767321167.076049:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3656 00000100:00100000:0.0:1767321167.076055:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985611904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321167.076085:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985611904:12345-0@lo:400:kworker.0 Request processed in 31us (538us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321167.076096:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3656 00000100:00100000:3.0:1767321167.081079:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985611776:0@lo:400:kworker.0 00000100:00100000:3.0:1767321167.081148:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321167.084282:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611008 00000100:00100000:2.0:1767321167.084306:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611008 00000100:00100000:2.0:1767321167.084393:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611520 00000100:00100000:2.0:1767321167.084397:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611520 00000100:00100000:2.0:1767321167.084463:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5659 00000100:00100000:2.0:1767321167.084475:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985611520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321167.084606:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985611520:12345-0@lo:400:kworker.0 Request processed in 132us (9171us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321167.084615:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5659 00000100:00100000:1.0:1767321167.084925:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6485 00000100:00100000:1.0:1767321167.084933:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985611008:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321167.084967:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321167.085018:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985611008:12345-0@lo:38:kworker.0 Request processed in 84us (12888us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321167.085026:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6485 00000100:00100000:2.0:1767321167.085401:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985611520:0@lo:400:kworker.0 00000100:00100000:3.0:1767321167.086416:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985611648:0@lo:38:kworker.0 00000100:00100000:3.0:1767321167.086491:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321167.086531:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321167.086536:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321167.086542:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985611008:0@lo:38:kworker.0 00000100:00080000:3.0:1767321167.086576:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321167.086580:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321167.086583:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985611136:0@lo:38:kworker.0 00000100:00100000:1.0:1767321167.086884:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321167.086936:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985611904:0@lo:400:kworker.0 00000100:00100000:2.0:1767321167.087641:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611392 00000100:00100000:2.0:1767321167.087652:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611392 00000100:00100000:2.0:1767321167.087713:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611648 00000100:00100000:2.0:1767321167.087717:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611648 00000100:00100000:2.0:1767321167.087902:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985611264 00000100:00100000:2.0:1767321167.087907:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985611264 00000100:00100000:2.0:1767321167.088030:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5660 00000100:00100000:2.0:1767321167.088112:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985611264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321167.088252:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985611264:12345-0@lo:400:kworker.0 Request processed in 210us (1366us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321167.088260:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5660 00000100:00100000:2.0:1767321167.088284:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6486 00000100:00100000:2.0:1767321167.088289:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985611392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321167.088313:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321167.088348:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985611392:12345-0@lo:38:kworker.0 Request processed in 57us (7200us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321167.088355:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6486 00000100:00100000:2.0:1767321167.088365:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6487 00000100:00100000:2.0:1767321167.088369:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985611648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321167.088378:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321167.088404:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985611648:12345-0@lo:38:kworker.0 Request processed in 33us (1914us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321167.088410:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6487 00000100:00080000:3.0:1767321167.088538:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321167.088546:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321167.088550:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985611392:0@lo:38:kworker.0 00000100:00080000:3.0:1767321167.088564:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321167.088568:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321167.088570:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985611648:0@lo:38:kworker.0 00000100:00100000:1.0:1767321167.088672:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985611264:0@lo:400:kworker.0 00010000:00010000:0.0:1767321167.710855:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321168.044361:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767321168.044465:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920658048 offset 0 length 520: 4 00000100:00100000:1.0:1767321168.044509:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321168.049215:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920658304 00000100:00100000:0.0:1767321168.049237:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920658304 00000100:00100000:0.0:1767321168.049351:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920658176 00000100:00100000:0.0:1767321168.049356:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920658176 00000100:00100000:0.0:1767321168.049386:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5661 00000100:00100000:0.0:1767321168.049398:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920658304:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321168.049511:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920658304:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 114us (5165us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321168.049520:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5661 00000100:00100000:0.0:1767321168.049543:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6488 00000100:00100000:0.0:1767321168.049548:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920658176:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321168.049572:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321168.049602:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920658176:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (5093us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321168.049609:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6488 00000100:00080000:1.0:1767321168.051048:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321168.610537:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321168.610577:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985612032:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321168.610691:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321168.610917:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612032 00000100:00100000:0.0:1767321168.610929:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612032 00000100:00100000:0.0:1767321168.611016:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2880 00000100:00100000:0.0:1767321168.611028:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985612032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321168.611074:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321168.611136:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985612032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (444us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321168.611145:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2880 00000100:00100000:0.0:1767321168.611243:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985612032:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321168.611547:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321168.611839:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321168.611856:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985612160:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321168.611922:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321168.612037:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612160 00000100:00100000:3.0:1767321168.612045:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612160 00000100:00100000:3.0:1767321168.612106:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2881 00000100:00100000:3.0:1767321168.612116:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985612160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321168.612139:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321168.612228:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985612160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (304us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321168.612235:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2881 00000100:00100000:3.0:1767321168.612278:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985612160:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321168.612407:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767321168.733835:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321169.758310:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321170.779838:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321171.804300:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321172.187506:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321172.187521:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321172.187525:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321172.187529:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321172.187535:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321172.187547:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6415 00000100:00080000:0.0:1767321172.187553:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321172.187562:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321172.187619:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3100 x1853163985612288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321172.187649:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321172.187653:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321172.187654:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321172.187658:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6415 00000100:00080000:0.0:1767321172.187661:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321172.187665:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321172.187682:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5180 x1853163985612416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321172.187772:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321172.187776:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321172.187778:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321172.187783:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6415 00000100:00080000:0.0:1767321172.187785:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321172.187791:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321172.187804:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b480 x1853163985612672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321172.187836:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321172.187839:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321172.187841:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321172.187845:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6415 00000100:00080000:0.0:1767321172.187847:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767321172.187849:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985612544:0@lo:400:kworker.0 00000100:00080000:0.0:1767321172.187851:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321172.187861:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89c00 x1853163985612928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321172.187904:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985612800:0@lo:400:kworker.0 00000100:00100000:3.0:1767321172.187976:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985612288:0@lo:38:kworker.0 00000100:00100000:1.0:1767321172.188089:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985613184:0@lo:400:kworker.0 00000100:00100000:0.0:1767321172.194609:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321172.194762:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321172.194781:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985613056:0@lo:400:kworker.0 00000100:00100000:0.0:1767321172.194838:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321172.194893:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613056 00000100:00100000:0.0:1767321172.194904:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613056 00000100:00100000:0.0:1767321172.195002:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3657 00000100:00100000:0.0:1767321172.195012:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985613056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321172.195085:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985613056:12345-0@lo:400:kworker.0 Request processed in 73us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321172.195094:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3657 00000100:00100000:0.0:1767321172.195842:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985613056:0@lo:400:kworker.0 00000100:00100000:2.0:1767321172.197650:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321172.197800:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612800 00000100:00100000:2.0:1767321172.197815:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612800 00000100:00100000:2.0:1767321172.197977:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612544 00000100:00100000:2.0:1767321172.197982:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612544 00000100:00100000:2.0:1767321172.198017:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5662 00000100:00100000:2.0:1767321172.198029:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985612800:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321172.198132:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985612800:12345-0@lo:400:kworker.0 Request processed in 104us (3537us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321172.198142:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5662 00000100:00100000:2.0:1767321172.198181:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5663 00000100:00100000:2.0:1767321172.198188:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985612544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321172.198222:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985612544:12345-0@lo:400:kworker.0 Request processed in 34us (581us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321172.198227:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5663 00000100:00100000:2.0:1767321172.198282:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985612544:0@lo:400:kworker.0 00000100:00100000:2.0:1767321172.198321:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985612800:0@lo:400:kworker.0 00000100:00100000:3.0:1767321172.198553:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321172.198592:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985612416:0@lo:38:kworker.0 00000100:00100000:3.0:1767321172.198627:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321172.198662:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985612672:0@lo:38:kworker.0 00000100:00100000:3.0:1767321172.198688:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321172.198712:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321172.198718:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612288 00000100:00080000:1.0:1767321172.198719:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321172.198722:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612288 00000100:00100000:3.0:1767321172.198744:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612672 00000100:00100000:3.0:1767321172.198747:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612672 00000100:00100000:3.0:1767321172.198759:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6489 00000100:00100000:1.0:1767321172.198760:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321172.198764:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985612288:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321172.198789:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321172.198813:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612416 00000100:00100000:1.0:1767321172.198818:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612416 00000100:00100000:3.0:1767321172.198822:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985612288:12345-0@lo:38:kworker.0 Request processed in 57us (270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321172.198828:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6489 00000100:00100000:3.0:1767321172.198841:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6490 00000100:00100000:3.0:1767321172.198845:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985612672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321172.198854:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321172.198878:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985612672:12345-0@lo:38:kworker.0 Request processed in 31us (189us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321172.198883:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6490 00000100:00080000:3.0:1767321172.198909:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321172.198914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321172.198918:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985612672:0@lo:38:kworker.0 00000100:00100000:3.0:1767321172.198926:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985612928:0@lo:38:kworker.0 00000100:00100000:1.0:1767321172.198939:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2113 00000100:00100000:1.0:1767321172.198946:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985612416:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321172.198957:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321172.198959:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321172.198979:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985612928 00000100:00100000:3.0:1767321172.198982:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985612928 00000100:00100000:1.0:1767321172.198985:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985612416:12345-0@lo:38:kworker.0 Request processed in 39us (357us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321172.198991:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2113 00000100:00100000:3.0:1767321172.198998:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6491 00000100:00100000:3.0:1767321172.199002:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985612928:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321172.199012:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321172.199016:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613184 00000100:00100000:1.0:1767321172.199019:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613184 00000100:00100000:3.0:1767321172.199036:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985612928:12345-0@lo:38:kworker.0 Request processed in 33us (76us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321172.199042:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6491 00000100:00100000:1.0:1767321172.199055:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3658 00000100:00100000:1.0:1767321172.199062:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985613184:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321172.199062:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321172.199066:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321172.199069:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985612928:0@lo:38:kworker.0 00000100:00080000:3.0:1767321172.199089:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321172.199094:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321172.199097:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985613184:12345-0@lo:400:kworker.0 Request processed in 36us (338us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321172.199097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985612288:0@lo:38:kworker.0 00000100:00080000:3.0:1767321172.199105:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321172.199108:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321172.199296:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3658 00000100:00080000:1.0:1767321172.199313:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321172.200497:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985613184:0@lo:400:kworker.0 00000100:00100000:3.0:1767321172.204333:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985612416:0@lo:38:kworker.0 00010000:00010000:2.0:1767321172.832474:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321173.169333:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321173.169411:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321173.169569:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920658560 00000100:00100000:2.0:1767321173.169582:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920658560 00000100:00100000:2.0:1767321173.169676:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920658432 00000100:00100000:2.0:1767321173.169709:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920658432 00000100:00100000:2.0:1767321173.169738:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5664 00000100:00100000:2.0:1767321173.169751:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920658560:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321173.169851:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920658560:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 103us (535us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321173.169859:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5664 00000100:00100000:2.0:1767321173.169882:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6492 00000100:00100000:2.0:1767321173.169887:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920658432:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321173.169914:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321173.169944:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920658432:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 55us (533us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321173.169951:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6492 00000100:00080000:1.0:1767321173.170197:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321173.723956:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321173.723990:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985613312:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321173.724094:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321173.724354:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613312 00000100:00100000:0.0:1767321173.724377:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613312 00000100:00100000:0.0:1767321173.724452:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2882 00000100:00100000:0.0:1767321173.724464:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985613312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321173.724502:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321173.724559:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985613312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (464us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321173.724567:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2882 00000100:00100000:1.0:1767321173.724886:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321173.724899:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985613440:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321173.724941:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321173.724971:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613440 00000100:00100000:1.0:1767321173.724975:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613440 00000100:00100000:1.0:1767321173.725003:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2883 00000100:00100000:1.0:1767321173.725011:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985613440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321173.725043:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321173.725076:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985613440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (134us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321173.725084:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2883 00000100:00100000:1.0:1767321173.725133:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985613440:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321173.725199:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985613312:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321173.853073:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321174.875473:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321175.901223:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321176.925062:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321177.307744:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321177.307758:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321177.307762:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321177.307767:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321177.307772:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321177.307785:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6420 00000100:00080000:2.0:1767321177.307792:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321177.307800:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321177.307854:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb100 x1853163985613568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321177.307878:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321177.307881:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321177.307882:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321177.307885:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6420 00000100:00080000:2.0:1767321177.307887:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321177.307889:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321177.307898:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9f80 x1853163985613696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321177.307961:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985613824:0@lo:400:kworker.0 00000100:00100000:1.0:1767321177.308059:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321177.308152:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613824 00000100:00100000:0.0:1767321177.308191:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613824 00000100:00080000:2.0:1767321177.308260:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321177.308265:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321177.308267:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767321177.308270:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5665 00000100:00080000:2.0:1767321177.308274:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6420 00000100:00080000:2.0:1767321177.308278:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321177.308285:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321177.308287:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985613824:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321177.308302:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9c00 x1853163985613952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321177.308333:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321177.308337:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321177.308339:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321177.308343:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6420 00000100:00080000:2.0:1767321177.308346:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767321177.308392:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985613568:0@lo:38:kworker.0 00000100:00100000:3.0:1767321177.308429:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321177.308459:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985613696:0@lo:38:kworker.0 00000100:00100000:3.0:1767321177.308488:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321177.308513:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985613952:0@lo:38:kworker.0 00000100:00100000:3.0:1767321177.308539:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321177.308573:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613568 00000100:00100000:3.0:1767321177.308577:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613568 00000100:00100000:3.0:1767321177.308613:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613952 00000100:00100000:3.0:1767321177.308617:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613952 00000100:00100000:3.0:1767321177.308637:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6493 00000100:00100000:3.0:1767321177.308642:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985613568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321177.308669:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321177.308708:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985613568:12345-0@lo:38:kworker.0 Request processed in 64us (277us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321177.308715:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6493 00000100:00100000:3.0:1767321177.308734:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6494 00000100:00100000:3.0:1767321177.308738:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985613952:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321177.308747:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321177.308772:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985613952:12345-0@lo:38:kworker.0 Request processed in 32us (232us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321177.308777:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6494 00000100:00080000:3.0:1767321177.308833:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321177.308838:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321177.308842:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985613568:0@lo:38:kworker.0 00000100:00080000:3.0:1767321177.308850:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321177.308853:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321177.308855:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985613952:0@lo:38:kworker.0 00000100:00100000:3.0:1767321177.308902:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321177.308912:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985614080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.308938:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321177.308963:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614080 00000100:00100000:3.0:1767321177.308966:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614080 00000100:00100000:3.0:1767321177.308995:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5666 00000100:00100000:3.0:1767321177.309001:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985614080:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.309036:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985614080:12345-0@lo:400:kworker.0 Request processed in 34us (97us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321177.309122:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5666 00000100:00100000:3.0:1767321177.309188:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985614080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.309217:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985613696 00000100:00100000:3.0:1767321177.309221:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985613696 00000100:00100000:3.0:1767321177.309245:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2114 00000100:00100000:3.0:1767321177.309250:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985613696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321177.309261:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321177.309294:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985613696:12345-0@lo:38:kworker.0 Request processed in 43us (805us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321177.309300:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2114 00000100:00080000:3.0:1767321177.309322:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321177.309327:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321177.309330:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985613696:0@lo:38:kworker.0 00000100:00080000:3.0:1767321177.309357:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321177.309491:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321177.309517:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd8a80 x1853163985614208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321177.309621:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985614208:0@lo:38:kworker.0 00000100:00100000:3.0:1767321177.309652:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321177.309682:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614208 00000100:00100000:3.0:1767321177.309686:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614208 00000100:00100000:3.0:1767321177.309706:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6495 00000100:00100000:3.0:1767321177.309710:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985614208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321177.309721:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321177.309752:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985614208:12345-0@lo:38:kworker.0 Request processed in 40us (99us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321177.309758:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6495 00000100:00080000:3.0:1767321177.309783:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321177.309787:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321177.309790:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985614208:0@lo:38:kworker.0 00000100:00100000:3.0:1767321177.309815:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985614336:0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.309845:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321177.309881:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985614464:0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.309909:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321177.309946:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614336 00000100:00100000:3.0:1767321177.309950:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614336 00000100:00100000:3.0:1767321177.309985:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3659 00000100:00100000:3.0:1767321177.309992:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985614336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.310028:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985614336:12345-0@lo:400:kworker.0 Request processed in 37us (182us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321177.310149:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3659 00000100:00100000:3.0:1767321177.310187:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614464 00000100:00100000:3.0:1767321177.310191:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614464 00000100:00100000:3.0:1767321177.310217:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3660 00000100:00100000:3.0:1767321177.310224:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985614464:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.310260:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985614464:12345-0@lo:400:kworker.0 Request processed in 36us (350us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321177.310266:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3660 00000100:00080000:3.0:1767321177.310281:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321177.310284:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321177.310301:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985614464:0@lo:400:kworker.0 00000100:00100000:3.0:1767321177.310322:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985614336:0@lo:400:kworker.0 00000100:00100000:0.0:1767321177.312505:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985613824:12345-0@lo:400:kworker.0 Request processed in 4212us (4433us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321177.312520:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5665 00000100:00100000:1.0:1767321177.312620:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985613824:0@lo:400:kworker.0 00010000:00010000:2.0:1767321177.947497:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321178.282546:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321178.282907:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321178.294721:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920658816 00000100:00100000:0.0:1767321178.294744:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920658816 00000100:00080000:2.0:1767321178.295012:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321178.295053:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920658688 00000100:00100000:2.0:1767321178.295058:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920658688 00000100:00100000:2.0:1767321178.295098:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5667 00000100:00100000:2.0:1767321178.295111:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920658816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321178.295265:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920658816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 156us (12735us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321178.295274:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5667 00000100:00100000:2.0:1767321178.295299:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6496 00000100:00100000:2.0:1767321178.295304:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920658688:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321178.295330:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321178.295361:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920658688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 55us (12456us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321178.295368:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6496 00000100:00100000:0.0:1767321178.844824:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321178.844857:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985614592:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321178.844961:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321178.845211:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321178.845225:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985614720:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321178.845261:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321178.845298:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614592 00000100:00100000:3.0:1767321178.845308:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614592 00000100:00100000:3.0:1767321178.845365:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614720 00000100:00100000:3.0:1767321178.845368:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614720 00000100:00100000:3.0:1767321178.845393:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2884 00000100:00100000:3.0:1767321178.845402:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985614592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321178.845441:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321178.845490:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985614592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (529us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321178.845499:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2884 00000100:00100000:3.0:1767321178.845806:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2885 00000100:00100000:3.0:1767321178.845815:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985614720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321178.845836:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321178.845872:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985614720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (609us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321178.845878:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2885 00000100:00100000:3.0:1767321178.845936:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985614720:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321178.846461:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985614592:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321178.971483:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321179.997348:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321181.019612:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321182.046129:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321182.430229:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321182.430246:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321182.430250:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321182.430255:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321182.430262:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321182.430273:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6425 00000100:00080000:0.0:1767321182.430279:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321182.430287:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321182.430344:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8ad80 x1853163985614848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321182.430458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985614848:0@lo:38:kworker.0 00000100:00100000:3.0:1767321182.430571:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321182.430748:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321182.430753:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321182.430755:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321182.430760:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6425 00000100:00080000:0.0:1767321182.430763:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321182.430767:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321182.430785:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88a80 x1853163985614976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321182.430856:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614848 00000100:00100000:1.0:1767321182.430866:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614848 00000100:00100000:1.0:1767321182.430921:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6497 00000100:00100000:1.0:1767321182.430930:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985614848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321182.430957:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321182.431414:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985614976:0@lo:38:kworker.0 00000100:00100000:3.0:1767321182.431495:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321182.431746:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321182.431751:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321182.431754:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321182.431758:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6425 00000100:00080000:0.0:1767321182.431761:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321182.431765:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321182.431781:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985615232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321182.431812:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321182.431815:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321182.431818:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321182.431822:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6425 00000100:00080000:0.0:1767321182.431824:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321182.431828:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321182.431839:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89f80 x1853163985615488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321182.431885:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985615744:0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.431923:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321182.431949:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321182.431955:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985615104:0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.431980:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321182.432009:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985615360:0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432034:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321182.432057:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321182.432061:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985615616:0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432082:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321182.432111:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985615744 00000100:00100000:0.0:1767321182.432117:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985615744 00000100:00100000:0.0:1767321182.432193:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985615616 00000100:00100000:0.0:1767321182.432197:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985615616 00000100:00100000:0.0:1767321182.432235:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3661 00000100:00100000:0.0:1767321182.432244:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985615744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432300:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985615744:12345-0@lo:400:kworker.0 Request processed in 56us (375us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321182.432308:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3661 00000100:00100000:0.0:1767321182.432329:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3662 00000100:00100000:0.0:1767321182.432335:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985615616:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432369:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985615616:12345-0@lo:400:kworker.0 Request processed in 35us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321182.432375:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3662 00000100:00100000:0.0:1767321182.432393:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985615104 00000100:00100000:0.0:1767321182.432397:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985615104 00000100:00100000:0.0:1767321182.432426:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985615360 00000100:00100000:0.0:1767321182.432428:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985615360 00000100:00100000:0.0:1767321182.432446:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5668 00000100:00100000:0.0:1767321182.432464:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985615104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432496:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985615104:12345-0@lo:400:kworker.0 Request processed in 44us (515us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321182.432502:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5668 00000100:00100000:0.0:1767321182.432527:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5669 00000100:00100000:0.0:1767321182.432534:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985615360:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432563:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985615360:12345-0@lo:400:kworker.0 Request processed in 29us (528us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321182.432568:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5669 00000100:00080000:0.0:1767321182.432592:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321182.432597:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321182.432599:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321182.432658:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985615616:0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432670:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985615360:0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432703:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985615104:0@lo:400:kworker.0 00000100:00100000:0.0:1767321182.432713:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985615744:0@lo:400:kworker.0 00000100:00100000:2.0:1767321182.432968:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985614976 00000100:00100000:2.0:1767321182.432974:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985614976 00000100:00100000:2.0:1767321182.433021:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2115 00000100:00100000:2.0:1767321182.433029:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985614976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321182.433050:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321182.433095:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985614976:12345-0@lo:38:kworker.0 Request processed in 66us (1643us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321182.433101:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2115 00000100:00100000:1.0:1767321182.433809:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985614848:12345-0@lo:38:kworker.0 Request processed in 2874us (3233us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321182.433821:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6497 00000100:00080000:3.0:1767321182.433903:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321182.433910:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321182.433915:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985614848:0@lo:38:kworker.0 00000100:00100000:3.0:1767321182.433943:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985615232:0@lo:38:kworker.0 00000100:00100000:3.0:1767321182.433990:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321182.434063:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985615488:0@lo:38:kworker.0 00000100:00100000:3.0:1767321182.434086:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321182.434158:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321182.434180:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321182.434184:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985614976:0@lo:38:kworker.0 00000100:00100000:1.0:1767321182.434367:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985615232 00000100:00100000:1.0:1767321182.434374:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985615232 00000100:00100000:1.0:1767321182.434412:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985615488 00000100:00100000:1.0:1767321182.434415:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985615488 00000100:00100000:1.0:1767321182.434438:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6498 00000100:00100000:1.0:1767321182.434445:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985615232:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321182.434478:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321182.434527:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985615232:12345-0@lo:38:kworker.0 Request processed in 81us (536us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321182.434534:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6498 00000100:00100000:1.0:1767321182.434547:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6499 00000100:00100000:1.0:1767321182.434551:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985615488:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321182.434560:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321182.434582:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985615488:12345-0@lo:38:kworker.0 Request processed in 30us (495us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321182.434589:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6499 00000100:00080000:3.0:1767321182.434797:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321182.434803:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321182.434807:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985615232:0@lo:38:kworker.0 00000100:00080000:3.0:1767321182.434816:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321182.434820:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321182.434823:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985615488:0@lo:38:kworker.0 00010000:00010000:1.0:1767321183.071409:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321183.405988:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321183.406049:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321183.406242:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920658944 00000100:00100000:0.0:1767321183.406257:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920658944 00000100:00100000:0.0:1767321183.406319:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920659072 00000100:00100000:0.0:1767321183.406323:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920659072 00000100:00100000:0.0:1767321183.406386:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5670 00000100:00100000:0.0:1767321183.406399:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920659072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767321183.406457:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321183.406538:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920659072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 141us (488us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321183.406547:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5670 00000100:00100000:0.0:1767321183.406572:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6500 00000100:00100000:0.0:1767321183.406577:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920658944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321183.406604:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321183.406715:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920658944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 135us (745us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321183.406722:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6500 00000100:00100000:0.0:1767321183.963900:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985615872:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321183.964026:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321183.964036:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985616000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321183.964082:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321183.964134:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985615872 00000100:00100000:0.0:1767321183.964145:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985615872 00000100:00100000:3.0:1767321183.964198:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616000 00000100:00100000:3.0:1767321183.964203:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616000 00000100:00100000:0.0:1767321183.964236:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2886 00000100:00100000:0.0:1767321183.964247:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985615872:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321183.964289:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321183.964344:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985615872:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (319us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321183.964353:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2886 00000100:00100000:0.0:1767321183.964375:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2887 00000100:00100000:0.0:1767321183.964381:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985616000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321183.964401:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321183.964432:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985616000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (349us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321183.964439:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2887 00000100:00100000:3.0:1767321183.965366:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985615872:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321183.965758:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985616000:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321184.092222:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321185.121970:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321186.140738:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321187.164373:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321187.548674:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321187.548685:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321187.548688:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321187.548692:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321187.548696:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321187.548705:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6430 00000100:00080000:2.0:1767321187.548710:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321187.548718:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321187.548761:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb800 x1853163985616128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321187.548791:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321187.548796:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321187.548798:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321187.548801:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6430 00000100:00080000:2.0:1767321187.548803:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321187.548806:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321187.548820:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9c00 x1853163985616256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321187.548866:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321187.548870:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321187.548872:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321187.548875:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6430 00000100:00080000:2.0:1767321187.548878:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321187.548881:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321187.548894:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9500 x1853163985616512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321187.548921:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321187.548924:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321187.548926:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321187.548929:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6430 00000100:00080000:2.0:1767321187.548932:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321187.548936:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321187.548948:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd8380 x1853163985616768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321187.548996:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985616640:0@lo:400:kworker.0 00000100:00100000:2.0:1767321187.549061:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321187.549136:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321187.549142:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985616896:0@lo:400:kworker.0 00000100:00100000:2.0:1767321187.549200:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321187.549234:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616640 00000100:00100000:2.0:1767321187.549241:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616640 00000100:00100000:3.0:1767321187.549392:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985616384:0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.549432:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321187.549460:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321187.549465:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985617024:0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.549486:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321187.549517:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616896 00000100:00100000:3.0:1767321187.549521:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616896 00000100:00100000:3.0:1767321187.549589:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617024 00000100:00100000:3.0:1767321187.549593:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617024 00000100:00100000:3.0:1767321187.549614:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3663 00000100:00100000:3.0:1767321187.549622:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985616896:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.549677:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985616896:12345-0@lo:400:kworker.0 Request processed in 55us (475us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321187.549684:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3663 00000100:00100000:3.0:1767321187.549699:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3664 00000100:00100000:3.0:1767321187.549705:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985617024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.549734:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985617024:12345-0@lo:400:kworker.0 Request processed in 29us (246us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321187.549740:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3664 00000100:00100000:3.0:1767321187.549779:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985617024:0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.549815:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616384 00000100:00100000:3.0:1767321187.549818:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616384 00000100:00100000:3.0:1767321187.549844:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5671 00000100:00100000:3.0:1767321187.549851:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985616384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.549884:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985616384:12345-0@lo:400:kworker.0 Request processed in 33us (451us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321187.549890:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5671 00000100:00100000:3.0:1767321187.549916:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985616384:0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.549942:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985616896:0@lo:400:kworker.0 00000100:00080000:1.0:1767321187.550183:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321187.550188:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321187.550191:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321187.550297:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985616128:0@lo:38:kworker.0 00000100:00100000:0.0:1767321187.550331:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321187.550368:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985616256:0@lo:38:kworker.0 00000100:00100000:0.0:1767321187.550401:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321187.550441:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985616512:0@lo:38:kworker.0 00000100:00100000:0.0:1767321187.550462:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321187.550478:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985616768:0@lo:38:kworker.0 00000100:00100000:0.0:1767321187.550496:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321187.550529:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616128 00000100:00100000:0.0:1767321187.550533:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616128 00000100:00100000:1.0:1767321187.550672:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616512 00000100:00100000:1.0:1767321187.550677:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616512 00000100:00100000:1.0:1767321187.550700:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616768 00000100:00100000:1.0:1767321187.550703:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616768 00000100:00100000:1.0:1767321187.550721:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6501 00000100:00100000:1.0:1767321187.550726:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985616512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321187.550745:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321187.550783:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985616512:12345-0@lo:38:kworker.0 Request processed in 56us (320us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321187.550790:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6501 00000100:00100000:1.0:1767321187.550804:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6502 00000100:00100000:1.0:1767321187.550807:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985616768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321187.550816:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321187.550838:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985616768:12345-0@lo:38:kworker.0 Request processed in 29us (340us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321187.550843:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6502 00000100:00080000:1.0:1767321187.550877:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321187.550882:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321187.550885:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985616512:0@lo:38:kworker.0 00000100:00080000:1.0:1767321187.550894:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321187.550897:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321187.550900:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985616768:0@lo:38:kworker.0 00000100:00100000:0.0:1767321187.551091:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6503 00000100:00100000:0.0:1767321187.551095:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985616128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321187.551105:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321187.551137:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985616128:12345-0@lo:38:kworker.0 Request processed in 40us (805us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321187.551143:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6503 00000100:00080000:0.0:1767321187.551189:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321187.551193:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321187.551196:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985616128:0@lo:38:kworker.0 00000100:00100000:2.0:1767321187.552002:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5672 00000100:00100000:2.0:1767321187.552011:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985616640:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321187.552067:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985616640:12345-0@lo:400:kworker.0 Request processed in 57us (3006us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321187.552073:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5672 00000100:00100000:2.0:1767321187.552111:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985616640:0@lo:400:kworker.0 00000100:00100000:3.0:1767321187.552201:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985616256 00000100:00100000:3.0:1767321187.552207:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985616256 00000100:00100000:3.0:1767321187.552237:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2116 00000100:00100000:3.0:1767321187.552243:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985616256:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321187.552266:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321187.552303:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985616256:12345-0@lo:38:kworker.0 Request processed in 59us (1900us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321187.552309:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2116 00000100:00080000:1.0:1767321187.552650:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321187.553716:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321187.553727:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321187.553731:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985616256:0@lo:38:kworker.0 00010000:00010000:2.0:1767321188.189511:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321188.522272:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321188.522363:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321188.522622:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920659328 00000100:00100000:3.0:1767321188.522632:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920659328 00000100:00080000:1.0:1767321188.522700:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321188.522719:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920659200 00000100:00100000:3.0:1767321188.522724:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920659200 00000100:00100000:3.0:1767321188.522751:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5673 00000100:00100000:3.0:1767321188.522762:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920659328:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321188.522796:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6504 00000100:00100000:1.0:1767321188.522804:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920659200:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321188.522837:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321188.522840:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920659328:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 78us (575us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321188.522850:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5673 00000100:00100000:1.0:1767321188.522872:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920659200:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 65us (506us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321188.522879:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6504 00000100:00100000:1.0:1767321189.083472:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985617280:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321189.083558:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321189.083604:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985617152:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321189.083632:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321189.083723:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617280 00000100:00100000:0.0:1767321189.083731:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617280 00000100:00100000:0.0:1767321189.083804:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617152 00000100:00100000:0.0:1767321189.083808:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617152 00000100:00100000:0.0:1767321189.083829:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2888 00000100:00100000:0.0:1767321189.083838:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985617280:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321189.083866:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321189.083913:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985617280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (355us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321189.083921:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2888 00000100:00100000:3.0:1767321189.083952:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2889 00000100:00100000:3.0:1767321189.083964:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985617152:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321189.083980:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:3.0:1767321189.083982:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767321189.083986:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321189.084021:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985617152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (387us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321189.084048:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985617280:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321189.084064:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985617152:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321189.086330:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2889 00010000:00010000:1.0:1767321189.211504:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321190.235388:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321191.261090:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321192.283372:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767321192.628379:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920659456 offset 0 length 520: 4 00000100:00100000:0.0:1767321192.628429:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321192.628500:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321192.628622:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920659584 00000100:00100000:2.0:1767321192.628633:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920659584 00000100:00100000:2.0:1767321192.628676:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920659712 00000100:00100000:2.0:1767321192.628679:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920659712 00000100:00100000:2.0:1767321192.628728:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5674 00000100:00100000:2.0:1767321192.628738:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920659712:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321192.628810:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920659712:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 74us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321192.628818:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5674 00000100:00100000:2.0:1767321192.628846:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6505 00000100:00100000:2.0:1767321192.628851:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920659584:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321192.628871:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321192.628897:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920659584:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (469us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321192.628903:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6505 00000100:00080000:0.0:1767321192.629572:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321192.667373:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.667383:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321192.667386:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.667390:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.667393:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.667401:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6435 00000100:00080000:0.0:1767321192.667405:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.667412:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321192.667454:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b89f80 x1853163985617408/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321192.667470:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.667473:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.667475:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.667478:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6435 00000100:00080000:0.0:1767321192.667480:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.667483:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321192.667494:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8aa00 x1853163985617536/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321192.667539:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.667543:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.667545:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.667549:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6435 00000100:00080000:0.0:1767321192.667552:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.667555:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321192.667571:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8ad80 x1853163985617792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321192.667586:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.667589:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.667591:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.667594:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6435 00000100:00080000:0.0:1767321192.667596:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.667598:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321192.667606:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5180 x1853163985618048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321192.667661:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985618176:0@lo:400:kworker.0 00000100:00100000:0.0:1767321192.667723:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321192.667790:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321192.667796:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985618304:0@lo:400:kworker.0 00000100:00100000:0.0:1767321192.667817:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321192.668041:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985617920:0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.668075:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321192.668104:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321192.668110:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985617664:0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.668131:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321192.668184:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617920 00000100:00100000:1.0:1767321192.668191:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617920 00000100:00100000:1.0:1767321192.668238:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617664 00000100:00100000:1.0:1767321192.668242:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617664 00000100:00100000:1.0:1767321192.668263:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5675 00000100:00100000:1.0:1767321192.668271:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985617920:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.668330:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985617920:12345-0@lo:400:kworker.0 Request processed in 49us (243us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321192.668345:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5675 00000100:00100000:1.0:1767321192.668360:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5676 00000100:00100000:1.0:1767321192.668366:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985617664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.668393:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985617664:12345-0@lo:400:kworker.0 Request processed in 27us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321192.668398:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5676 00000100:00080000:1.0:1767321192.668414:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321192.668444:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985617664:0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.668454:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985617920:0@lo:400:kworker.0 00000100:00100000:2.0:1767321192.668632:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321167/real 1767321167] req@ffff8ce1b3ffdf80 x1853163985610880/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321192 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321192.668649:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3ffdf80 x1853163985610880/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321192 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321192.668662:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985617408:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.668689:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.668722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985617536:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.668745:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.668767:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985617792:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.668787:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.668800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985618048:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.668819:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321192.668847:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321192.668860:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321192.668864:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985610880:0@lo:250:kworker.0 00000100:00100000:2.0:1767321192.668892:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617536 00000100:00100000:2.0:1767321192.668895:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617536 00000100:00100000:2.0:1767321192.668915:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2117 00000100:00100000:2.0:1767321192.668920:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985617536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321192.668938:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321192.668969:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985617536:12345-0@lo:38:kworker.0 Request processed in 48us (224us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321192.668975:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2117 00000100:00100000:2.0:1767321192.668993:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617408 00000100:00100000:2.0:1767321192.668996:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617408 00000100:00100000:2.0:1767321192.669013:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985617792 00000100:00100000:2.0:1767321192.669015:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985617792 00000100:00100000:2.0:1767321192.669027:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985618048 00000100:00100000:2.0:1767321192.669029:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985618048 00000100:00100000:2.0:1767321192.669038:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6506 00000100:00100000:2.0:1767321192.669042:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985617408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321192.669052:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321192.669075:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985617408:12345-0@lo:38:kworker.0 Request processed in 31us (385us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321192.669080:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6506 00000100:00100000:2.0:1767321192.669091:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6507 00000100:00100000:2.0:1767321192.669095:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985617792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321192.669103:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321192.669122:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985617792:12345-0@lo:38:kworker.0 Request processed in 26us (334us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321192.669127:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6507 00000100:00100000:2.0:1767321192.669137:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6508 00000100:00100000:2.0:1767321192.669141:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985618048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321192.669149:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321192.669188:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985618048:12345-0@lo:38:kworker.0 Request processed in 45us (368us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321192.669194:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6508 00000100:00080000:2.0:1767321192.669217:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321192.669221:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321192.669224:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985617408:0@lo:38:kworker.0 00000100:00080000:2.0:1767321192.669232:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321192.669235:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321192.669237:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985617536:0@lo:38:kworker.0 00000100:00080000:2.0:1767321192.669244:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321192.669246:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321192.669249:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985617792:0@lo:38:kworker.0 00000100:00080000:2.0:1767321192.669255:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321192.669258:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321192.669260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985618048:0@lo:38:kworker.0 00000100:00080000:2.0:1767321192.669283:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321192.669287:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767321192.669289:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321192.669292:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6415 00000100:00080000:2.0:1767321192.669295:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321192.669300:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321192.669327:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f04000 x1853163985618432/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321192.669358:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985618432:0@lo:250:kworker.0 00000400:00000100:2.0:1767321192.669376:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985618432 offset 0 length 520: 4 00000100:00100000:1.0:1767321192.675423:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985618176 00000100:00100000:1.0:1767321192.675441:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985618176 00000100:00100000:1.0:1767321192.675546:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985618304 00000100:00100000:1.0:1767321192.675550:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985618304 00000100:00100000:1.0:1767321192.675590:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3665 00000100:00100000:1.0:1767321192.675601:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985618176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.675711:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985618176:12345-0@lo:400:kworker.0 Request processed in 112us (7987us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321192.675719:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3665 00000100:00100000:1.0:1767321192.675735:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3666 00000100:00100000:1.0:1767321192.675741:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985618304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.675768:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985618304:12345-0@lo:400:kworker.0 Request processed in 28us (7949us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321192.675773:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3666 00000100:00080000:0.0:1767321192.676094:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321192.676100:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321192.676143:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985618304:0@lo:400:kworker.0 00000100:00100000:0.0:1767321192.676155:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985618176:0@lo:400:kworker.0 00010000:00010000:1.0:1767321193.307339:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321194.203622:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985618688:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321194.203703:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321194.203756:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321194.203763:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985618560:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321194.203785:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321194.203864:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985618688 00000100:00100000:0.0:1767321194.203874:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985618688 00000100:00100000:0.0:1767321194.203942:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985618560 00000100:00100000:0.0:1767321194.203945:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985618560 00000100:00100000:0.0:1767321194.203971:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2890 00000100:00100000:0.0:1767321194.203981:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985618688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321194.204021:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321194.204074:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985618688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (370us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321194.204082:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2890 00000100:00100000:0.0:1767321194.204101:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2891 00000100:00100000:0.0:1767321194.204107:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985618560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321194.204125:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321194.204151:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985618560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 45us (365us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321194.204156:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2891 00000100:00080000:1.0:1767321194.204493:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321194.204499:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321194.204536:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985618560:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321194.204551:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985618688:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321194.331504:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321195.355462:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321196.379409:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321197.403433:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321197.739025:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321197.739265:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920659968 00000100:00100000:3.0:1767321197.739276:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920659968 00000100:00100000:3.0:1767321197.739353:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5677 00000100:00100000:3.0:1767321197.739363:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920659968:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321197.739514:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920659968:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 75us (421us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321197.739525:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5677 00000100:00080000:1.0:1767321197.739596:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321197.739808:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321197.739852:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920659840 00000100:00100000:2.0:1767321197.739856:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920659840 00000100:00100000:2.0:1767321197.739888:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6509 00000100:00100000:2.0:1767321197.739894:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920659840:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321197.739913:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321197.739943:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920659840:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (135us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321197.739949:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6509 00000100:00080000:0.0:1767321197.787672:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321197.787684:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321197.787687:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321197.787691:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321197.787695:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321197.787704:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6440 00000100:00080000:0.0:1767321197.787709:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321197.787715:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321197.787765:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5880 x1853163985618816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321197.787819:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985618816:0@lo:38:kworker.0 00000100:00100000:2.0:1767321197.787897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321197.788055:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985618816 00000100:00100000:3.0:1767321197.788072:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985618816 00000100:00100000:3.0:1767321197.788107:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6510 00000100:00100000:3.0:1767321197.788115:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985618816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321197.788136:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321197.788648:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321197.788654:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321197.788657:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321197.788661:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6440 00000100:00080000:0.0:1767321197.788664:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321197.788668:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321197.788688:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5c00 x1853163985618944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321197.792939:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985618944:0@lo:38:kworker.0 00000100:00100000:2.0:1767321197.793019:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321197.793184:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985618944 00000100:00100000:1.0:1767321197.793192:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985618944 00000100:00100000:1.0:1767321197.793241:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2118 00000100:00100000:1.0:1767321197.793249:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985618944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321197.793272:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321197.793316:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985618944:12345-0@lo:38:kworker.0 Request processed in 66us (297us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321197.793323:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2118 00000100:00100000:3.0:1767321197.793600:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985618816:12345-0@lo:38:kworker.0 Request processed in 5481us (5700us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321197.793608:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6510 00000100:00080000:2.0:1767321197.793849:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321197.793855:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321197.793859:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985618816:0@lo:38:kworker.0 00000100:00080000:2.0:1767321197.793885:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321197.793888:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321197.793891:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985618944:0@lo:38:kworker.0 00000100:00100000:2.0:1767321197.794255:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:1.0:1767321197.794315:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321197.794320:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321197.794323:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321197.794327:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6440 00000100:00080000:1.0:1767321197.794331:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321197.794336:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321197.794357:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631880 x1853163985619200/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321197.794421:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985619200:0@lo:38:kworker.0 00000100:00100000:3.0:1767321197.794467:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321197.794543:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619200 00000100:00100000:0.0:1767321197.794550:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619200 00000100:00100000:0.0:1767321197.794586:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6511 00000100:00100000:0.0:1767321197.794592:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985619200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321197.794609:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321197.794643:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985619200:12345-0@lo:38:kworker.0 Request processed in 49us (175us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321197.794650:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6511 00000100:00080000:3.0:1767321197.794723:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321197.794727:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321197.794730:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985619200:0@lo:38:kworker.0 00000100:00100000:2.0:1767321197.798408:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985619328:0@lo:400:kworker.0 00000100:00100000:2.0:1767321197.798501:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321197.798625:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985619072:0@lo:400:kworker.0 00000100:00100000:2.0:1767321197.798647:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321197.798725:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321197.798729:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321197.798734:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321197.798749:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6440 00000100:00080000:1.0:1767321197.798753:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321197.798758:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321197.798791:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631880 x1853163985619456/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321197.798836:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321197.798845:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985619584:0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.798876:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321197.798928:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985619712:0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.798949:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321197.799010:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619584 00000100:00100000:1.0:1767321197.799018:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619584 00000100:00100000:1.0:1767321197.799078:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619712 00000100:00100000:1.0:1767321197.799081:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619712 00000100:00100000:1.0:1767321197.799104:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3667 00000100:00100000:1.0:1767321197.799113:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985619584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.799162:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985619584:12345-0@lo:400:kworker.0 Request processed in 49us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321197.799187:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3667 00000100:00100000:1.0:1767321197.799202:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3668 00000100:00100000:1.0:1767321197.799207:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985619712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.799245:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985619712:12345-0@lo:400:kworker.0 Request processed in 38us (295us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321197.799251:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3668 00000100:00080000:1.0:1767321197.799272:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321197.799276:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321197.799308:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985619712:0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.799318:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985619584:0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.799354:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985619456:0@lo:38:kworker.0 00000100:00100000:1.0:1767321197.799933:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321197.799954:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619328 00000100:00100000:3.0:1767321197.799959:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619328 00000100:00100000:0.0:1767321197.800017:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619072 00000100:00100000:0.0:1767321197.800021:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619072 00000100:00100000:0.0:1767321197.800052:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5678 00000100:00100000:0.0:1767321197.800070:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985619328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321197.800112:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985619328:12345-0@lo:400:kworker.0 Request processed in 53us (1610us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321197.800118:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5678 00000100:00100000:0.0:1767321197.800132:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5679 00000100:00100000:0.0:1767321197.800138:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985619072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321197.800186:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985619072:12345-0@lo:400:kworker.0 Request processed in 48us (1538us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321197.800192:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5679 00000100:00080000:2.0:1767321197.800261:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321197.800283:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985619328:0@lo:400:kworker.0 00000100:00100000:2.0:1767321197.800293:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985619072:0@lo:400:kworker.0 00000100:00100000:3.0:1767321197.800639:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619456 00000100:00100000:3.0:1767321197.800644:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619456 00000100:00100000:3.0:1767321197.800705:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6512 00000100:00100000:3.0:1767321197.800710:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985619456:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321197.800729:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321197.800779:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985619456:12345-0@lo:38:kworker.0 Request processed in 67us (847us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321197.800784:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6512 00000100:00080000:3.0:1767321197.800982:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321197.800987:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321197.800990:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985619456:0@lo:38:kworker.0 00010000:00010000:0.0:1767321198.427404:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321199.326423:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985619840:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321199.326507:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321199.326640:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619840 00000100:00100000:3.0:1767321199.326650:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619840 00000100:00100000:3.0:1767321199.326716:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2892 00000100:00100000:3.0:1767321199.326725:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985619840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321199.326757:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321199.326806:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985619840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (298us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321199.326813:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2892 00000100:00100000:3.0:1767321199.326859:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985619840:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321199.326976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321199.334032:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321199.334055:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985619968:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321199.334124:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321199.334464:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985619968 00000100:00100000:0.0:1767321199.334472:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985619968 00000100:00100000:0.0:1767321199.334583:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2893 00000100:00100000:0.0:1767321199.334592:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985619968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321199.334619:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321199.334674:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985619968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (549us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321199.334682:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2893 00000100:00100000:2.0:1767321199.334955:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985619968:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321199.335003:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321199.451489:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321200.481037:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321201.499621:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321202.525210:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321202.858427:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321202.858476:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321202.859640:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660224 00000100:00100000:3.0:1767321202.859652:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660224 00000100:00100000:3.0:1767321202.859729:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660096 00000100:00100000:3.0:1767321202.859734:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660096 00000100:00100000:3.0:1767321202.859892:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5680 00000100:00100000:3.0:1767321202.859904:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920660224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321202.860158:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6513 00000100:00100000:2.0:1767321202.860195:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920660096:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321202.860217:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321202.860246:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920660096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (1770us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321202.860254:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6513 00000100:00080000:1.0:1767321202.860305:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321202.860930:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920660224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (1562us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321202.860938:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5680 00000100:00080000:0.0:1767321202.907517:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321202.907528:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321202.907532:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321202.907536:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321202.907540:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321202.907548:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6446 00000100:00080000:0.0:1767321202.907553:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321202.907559:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321202.907601:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7b80 x1853163985620096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321202.907630:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321202.907633:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321202.907636:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321202.907639:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6446 00000100:00080000:0.0:1767321202.907641:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321202.907643:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321202.907656:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7480 x1853163985620224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321202.907748:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985620096:0@lo:38:kworker.0 00000100:00100000:3.0:1767321202.907748:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:0.0:1767321202.907767:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321202.907771:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321202.907772:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321202.907775:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6446 00000100:00080000:0.0:1767321202.907777:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321202.907780:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321202.907792:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5f80 x1853163985620480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321202.907808:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321202.907850:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321202.907858:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985620224:0@lo:38:kworker.0 00000100:00100000:2.0:1767321202.907859:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985620608:0@lo:400:kworker.0 00000100:00100000:1.0:1767321202.907883:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321202.907883:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321202.907910:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985620480:0@lo:38:kworker.0 00000100:00100000:2.0:1767321202.907918:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620096 00000100:00100000:2.0:1767321202.907926:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620096 00000100:00100000:1.0:1767321202.907931:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321202.907955:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620608 00000100:00100000:1.0:1767321202.907957:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620224 00000100:00100000:2.0:1767321202.907958:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620608 00000100:00100000:1.0:1767321202.907960:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620224 00000100:00100000:2.0:1767321202.907989:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620480 00000100:00100000:1.0:1767321202.907993:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2119 00000100:00100000:2.0:1767321202.907993:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620480 00000100:00100000:1.0:1767321202.908001:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985620224:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321202.908004:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5681 00000100:00100000:2.0:1767321202.908010:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985620608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.908062:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985620608:12345-0@lo:400:kworker.0 Request processed in 51us (176us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321202.908069:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5681 00000100:00100000:2.0:1767321202.908084:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6514 00000100:00100000:2.0:1767321202.908088:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985620096:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321202.908105:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767321202.908139:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321202.909113:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985620224:12345-0@lo:38:kworker.0 Request processed in 1109us (1226us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321202.909123:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2119 00000100:00100000:1.0:1767321202.909196:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6515 00000100:00100000:1.0:1767321202.909201:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985620480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321202.909214:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321202.909257:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985620480:12345-0@lo:38:kworker.0 Request processed in 54us (1324us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321202.909263:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6515 00000100:00080000:1.0:1767321202.909310:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321202.909658:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985620608:0@lo:400:kworker.0 00000100:00080000:1.0:1767321202.909705:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321202.909710:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321202.909713:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985620480:0@lo:38:kworker.0 00000100:00080000:1.0:1767321202.909723:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321202.909726:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321202.909729:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985620096:0@lo:38:kworker.0 00000100:00080000:1.0:1767321202.909737:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321202.909740:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321202.909743:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985620224:0@lo:38:kworker.0 00000100:00100000:3.0:1767321202.911954:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985620352:0@lo:400:kworker.0 00000100:00100000:3.0:1767321202.912290:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321202.912796:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620352 00000100:00100000:1.0:1767321202.912805:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620352 00000100:00100000:1.0:1767321202.912868:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5682 00000100:00100000:1.0:1767321202.912876:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985620352:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.914855:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985620096:12345-0@lo:38:kworker.0 Request processed in 6757us (7037us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767321202.914912:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767321202.914917:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985620352:12345-0@lo:400:kworker.0 Request processed in 2041us (2627us total) trans 0 rc 0/0 00000100:00080000:0.0:1767321202.914919:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321202.914922:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321202.914927:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6446 00000100:00100000:1.0:1767321202.914929:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5682 00000100:00080000:0.0:1767321202.914930:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321202.914937:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00080000:1.0:1767321202.914971:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321202.915239:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985620352:0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.916382:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6514 00000100:00100000:0.0:1767321202.916633:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985620736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321202.916867:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985620736:0@lo:38:kworker.0 00000100:00100000:1.0:1767321202.916930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321202.918126:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620736 00000100:00100000:0.0:1767321202.918145:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620736 00000100:00100000:0.0:1767321202.918215:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6516 00000100:00100000:0.0:1767321202.918223:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985620736:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321202.918244:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321202.918316:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985620736:12345-0@lo:38:kworker.0 Request processed in 92us (1385us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321202.918323:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6516 00000100:00080000:0.0:1767321202.918367:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321202.918372:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321202.918376:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985620736:0@lo:38:kworker.0 00000100:00100000:2.0:1767321202.918467:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985620864:0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.918505:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321202.918541:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321202.918548:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985620992:0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.918568:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321202.918596:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620864 00000100:00100000:2.0:1767321202.918600:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620864 00000100:00100000:2.0:1767321202.918637:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985620992 00000100:00100000:2.0:1767321202.918641:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985620992 00000100:00100000:2.0:1767321202.918662:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3669 00000100:00100000:2.0:1767321202.918670:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985620864:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.918711:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985620864:12345-0@lo:400:kworker.0 Request processed in 43us (206us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321202.918717:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3669 00000100:00100000:2.0:1767321202.918731:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3670 00000100:00100000:2.0:1767321202.918738:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985620992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.918765:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985620992:12345-0@lo:400:kworker.0 Request processed in 28us (196us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321202.918773:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3670 00000100:00080000:2.0:1767321202.918788:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321202.918791:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321202.918813:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985620992:0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.918823:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985620864:0@lo:400:kworker.0 00010000:00010000:2.0:1767321203.550738:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321204.444635:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985621120:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321204.444722:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321204.445128:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985621120 00000100:00100000:3.0:1767321204.445136:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985621120 00000100:00100000:3.0:1767321204.445213:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2894 00000100:00100000:3.0:1767321204.445222:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985621120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321204.445253:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321204.445297:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985621120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (584us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321204.445305:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2894 00000100:00100000:2.0:1767321204.445343:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985621120:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321204.445501:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321204.445616:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321204.445626:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985621248:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321204.445664:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321204.445881:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985621248 00000100:00100000:0.0:1767321204.445887:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985621248 00000100:00100000:0.0:1767321204.445928:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2895 00000100:00100000:0.0:1767321204.445942:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985621248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321204.445965:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321204.445995:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985621248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (341us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321204.446001:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2895 00000100:00080000:1.0:1767321204.446189:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321204.446463:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985621248:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767321204.571410:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321205.595398:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321206.619476:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321207.643437:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321207.988300:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321207.988421:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321207.988512:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660480 00000100:00100000:2.0:1767321207.988528:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660480 00000100:00100000:2.0:1767321207.988595:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660352 00000100:00100000:2.0:1767321207.988597:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660352 00000100:00100000:2.0:1767321207.988612:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5683 00000100:00100000:2.0:1767321207.988622:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920660480:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321207.988705:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920660480:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (432us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321207.988711:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5683 00000100:00100000:2.0:1767321207.988726:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6517 00000100:00100000:2.0:1767321207.988729:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920660352:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321207.988745:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321207.988763:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920660352:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 33us (343us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321207.988767:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6517 00000100:00080000:3.0:1767321207.990200:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321208.027320:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321208.027330:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321208.027333:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321208.027336:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321208.027340:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321208.027347:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6451 00000100:00080000:0.0:1767321208.027352:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321208.027358:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321208.027472:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7480 x1853163985621376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321208.027492:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321208.027495:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321208.027497:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321208.027500:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6451 00000100:00080000:0.0:1767321208.027503:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321208.027507:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321208.027520:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7b80 x1853163985621504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321208.027546:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321208.027549:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321208.027552:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321208.027556:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6451 00000100:00080000:0.0:1767321208.027558:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321208.027561:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321208.027571:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4700 x1853163985621760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321208.027586:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321208.027589:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321208.027591:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321208.027593:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6451 00000100:00080000:0.0:1767321208.027595:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767321208.027675:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985621888:0@lo:400:kworker.0 00000100:00100000:2.0:1767321208.027743:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321208.027804:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321208.027811:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985621632:0@lo:400:kworker.0 00000100:00100000:2.0:1767321208.027831:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321208.027914:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985621888 00000100:00100000:1.0:1767321208.027922:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985621888 00000100:00100000:3.0:1767321208.027931:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985621632 00000100:00100000:3.0:1767321208.027935:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985621632 00000100:00080000:0.0:1767321208.027997:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321208.028012:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5c00 x1853163985622016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321208.028069:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985622144:0@lo:400:kworker.0 00000100:00100000:0.0:1767321208.028100:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321208.028119:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321208.028124:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985622272:0@lo:400:kworker.0 00000100:00100000:0.0:1767321208.028144:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321208.028191:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622144 00000100:00100000:0.0:1767321208.028195:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622144 00000100:00100000:1.0:1767321208.028404:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5684 00000100:00100000:1.0:1767321208.028413:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985621888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321208.028470:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985621888:12345-0@lo:400:kworker.0 Request processed in 59us (727us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321208.028476:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5684 00000100:00100000:1.0:1767321208.028520:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985621888:0@lo:400:kworker.0 00000100:00100000:1.0:1767321208.028550:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622272 00000100:00100000:1.0:1767321208.028554:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622272 00000100:00080000:2.0:1767321208.030541:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321208.030550:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321208.030553:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321208.030630:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985621376:0@lo:38:kworker.0 00000100:00100000:2.0:1767321208.030708:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321208.030750:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985621504:0@lo:38:kworker.0 00000100:00100000:2.0:1767321208.030776:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321208.030799:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985621760:0@lo:38:kworker.0 00000100:00100000:2.0:1767321208.030818:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321208.030833:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985622016:0@lo:38:kworker.0 00000100:00100000:3.0:1767321208.030833:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985621376 00000100:00100000:3.0:1767321208.030840:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985621376 00000100:00100000:2.0:1767321208.030851:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321208.030857:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985621760 00000100:00100000:3.0:1767321208.030859:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985621760 00000100:00100000:3.0:1767321208.030867:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622016 00000100:00100000:3.0:1767321208.030870:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622016 00000100:00100000:3.0:1767321208.030879:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5685 00000100:00100000:2.0:1767321208.030880:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985621504 00000100:00100000:2.0:1767321208.030882:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985621504 00000100:00100000:3.0:1767321208.030889:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985621632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321208.030902:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2120 00000100:00100000:2.0:1767321208.030906:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985621504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321208.030924:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321208.030940:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985621632:12345-0@lo:400:kworker.0 Request processed in 53us (3107us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321208.030947:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5685 00000100:00100000:2.0:1767321208.030950:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985621504:12345-0@lo:38:kworker.0 Request processed in 43us (173us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321208.030962:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6518 00000100:00100000:3.0:1767321208.030966:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985621376:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321208.030969:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2120 00010000:00020000:3.0:1767321208.030976:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321208.030996:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6519 00000100:00100000:3.0:1767321208.030997:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985621376:12345-0@lo:38:kworker.0 Request processed in 30us (290us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321208.031000:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985621760:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321208.031003:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6518 00010000:00020000:2.0:1767321208.031009:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321208.031011:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6520 00000100:00100000:3.0:1767321208.031015:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985622016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321208.031023:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321208.031030:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985621760:12345-0@lo:38:kworker.0 Request processed in 28us (211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321208.031047:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6519 00000100:00100000:3.0:1767321208.031051:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985622016:12345-0@lo:38:kworker.0 Request processed in 35us (199us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321208.031056:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6520 00000100:00080000:2.0:1767321208.031087:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321208.031092:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321208.031095:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985621376:0@lo:38:kworker.0 00000100:00080000:2.0:1767321208.031104:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321208.031107:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321208.031109:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985621504:0@lo:38:kworker.0 00000100:00080000:2.0:1767321208.031115:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321208.031118:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321208.031120:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985621760:0@lo:38:kworker.0 00000100:00080000:2.0:1767321208.031126:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321208.031128:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321208.031131:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985622016:0@lo:38:kworker.0 00000100:00100000:3.0:1767321208.031199:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985621632:0@lo:400:kworker.0 00000100:00100000:0.0:1767321208.032249:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3671 00000100:00100000:0.0:1767321208.032261:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985622144:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321208.032588:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985622144:12345-0@lo:400:kworker.0 Request processed in 330us (4486us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321208.032598:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3671 00000100:00100000:3.0:1767321208.032677:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985622144:0@lo:400:kworker.0 00000100:00100000:1.0:1767321208.035894:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3672 00000100:00100000:1.0:1767321208.035912:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985622272:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321208.036008:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985622272:12345-0@lo:400:kworker.0 Request processed in 103us (7862us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321208.036016:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3672 00000100:00100000:3.0:1767321208.036100:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985622272:0@lo:400:kworker.0 00010000:00010000:2.0:1767321208.667528:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321209.563838:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985622400:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321209.563911:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321209.563969:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321209.563977:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985622528:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321209.564001:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321209.564221:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622400 00000100:00100000:0.0:1767321209.564229:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622400 00000100:00100000:0.0:1767321209.564289:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622528 00000100:00100000:0.0:1767321209.564292:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622528 00000100:00100000:0.0:1767321209.564316:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2896 00000100:00100000:0.0:1767321209.564325:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985622400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321209.564366:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321209.564409:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985622400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (497us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321209.564416:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2896 00000100:00100000:0.0:1767321209.564434:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2897 00000100:00100000:0.0:1767321209.564440:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb0450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985622528:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321209.564456:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321209.564491:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb0450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985622528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (478us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321209.564497:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2897 00000100:00100000:2.0:1767321209.565103:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985622528:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321209.565118:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985622400:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321209.691432:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321210.715956:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321211.742877:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321212.764088:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321213.100539:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321213.100690:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321213.115659:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660736 00000100:00100000:2.0:1767321213.115677:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660736 00000100:00100000:2.0:1767321213.115778:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660608 00000100:00100000:2.0:1767321213.115782:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660608 00000100:00100000:2.0:1767321213.115812:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5686 00000100:00100000:2.0:1767321213.115822:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920660736:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321213.115908:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920660736:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (15381us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321213.115916:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5686 00000100:00100000:2.0:1767321213.115937:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6521 00000100:00100000:2.0:1767321213.115972:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920660608:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321213.115995:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321213.116021:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920660608:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (15331us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321213.116027:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6521 00000100:00080000:3.0:1767321213.116237:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321213.154472:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321213.154483:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321213.154486:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321213.154493:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321213.154498:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321213.154506:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6456 00000100:00080000:2.0:1767321213.154511:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321213.154518:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321213.154559:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f05c00 x1853163985622656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321213.154576:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321213.154579:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321213.154581:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321213.154584:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6456 00000100:00080000:2.0:1767321213.154587:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321213.154590:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321213.154604:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f07800 x1853163985622784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321213.154658:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321213.154662:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321213.154664:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321213.154667:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6456 00000100:00080000:2.0:1767321213.154670:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321213.154673:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321213.154685:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffd880 x1853163985623040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321213.154700:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321213.154703:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321213.154705:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321213.154708:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6456 00000100:00080000:2.0:1767321213.154711:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321213.154713:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321213.154723:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffc380 x1853163985623296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321213.154781:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985623424:0@lo:400:kworker.0 00000100:00100000:2.0:1767321213.154843:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321213.154904:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321213.154910:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985623552:0@lo:400:kworker.0 00000100:00100000:2.0:1767321213.154931:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321213.154973:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985623424 00000100:00100000:2.0:1767321213.154981:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985623424 00000100:00100000:2.0:1767321213.155024:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985623552 00000100:00100000:2.0:1767321213.155027:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985623552 00000100:00100000:2.0:1767321213.155045:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3673 00000100:00100000:2.0:1767321213.155053:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985623424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321213.155110:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985623424:12345-0@lo:400:kworker.0 Request processed in 58us (266us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321213.155117:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3673 00000100:00100000:2.0:1767321213.155133:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3674 00000100:00100000:2.0:1767321213.155139:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985623552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321213.155162:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985623552:12345-0@lo:400:kworker.0 Request processed in 23us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321213.155193:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3674 00000100:00100000:2.0:1767321213.155236:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985623552:0@lo:400:kworker.0 00000100:00100000:2.0:1767321213.155246:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985623424:0@lo:400:kworker.0 00000100:00100000:3.0:1767321213.155933:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985622912:0@lo:400:kworker.0 00000100:00100000:3.0:1767321213.155982:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321213.156026:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321213.156033:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985623168:0@lo:400:kworker.0 00000100:00100000:3.0:1767321213.156053:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321213.156077:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321213.156081:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321213.157639:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622912 00000100:00100000:2.0:1767321213.157650:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622912 00000100:00100000:2.0:1767321213.157718:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985623168 00000100:00100000:2.0:1767321213.157721:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985623168 00000100:00100000:2.0:1767321213.157744:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5687 00000100:00100000:2.0:1767321213.157753:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985622912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321213.157824:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985622912:12345-0@lo:400:kworker.0 Request processed in 72us (1840us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321213.157830:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5687 00000100:00100000:2.0:1767321213.157846:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5688 00000100:00100000:2.0:1767321213.157852:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985623168:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321213.157878:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985623168:12345-0@lo:400:kworker.0 Request processed in 27us (1824us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321213.157884:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5688 00000100:00080000:3.0:1767321213.157981:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321213.158020:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985623168:0@lo:400:kworker.0 00000100:00100000:3.0:1767321213.158032:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985622912:0@lo:400:kworker.0 00000100:00100000:0.0:1767321213.158143:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985622656:0@lo:38:kworker.0 00000100:00100000:0.0:1767321213.158191:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321213.158362:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985622784:0@lo:38:kworker.0 00000100:00100000:0.0:1767321213.158392:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321213.158428:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985623040:0@lo:38:kworker.0 00000100:00100000:0.0:1767321213.158450:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321213.158464:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985623296:0@lo:38:kworker.0 00000100:00100000:0.0:1767321213.158485:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321213.158558:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622784 00000100:00100000:3.0:1767321213.158562:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622784 00000100:00100000:3.0:1767321213.158595:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2121 00000100:00100000:3.0:1767321213.158601:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985622784:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321213.158619:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321213.158655:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985622784:12345-0@lo:38:kworker.0 Request processed in 53us (262us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321213.158662:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2121 00000100:00100000:2.0:1767321213.158724:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985622656 00000100:00100000:2.0:1767321213.158728:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985622656 00000100:00100000:2.0:1767321213.158755:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985623040 00000100:00100000:2.0:1767321213.158758:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985623040 00000100:00100000:2.0:1767321213.158770:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985623296 00000100:00100000:2.0:1767321213.158772:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985623296 00000100:00100000:2.0:1767321213.158783:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6522 00000100:00100000:2.0:1767321213.158787:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985622656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321213.158796:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321213.158823:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985622656:12345-0@lo:38:kworker.0 Request processed in 34us (631us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321213.158832:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6522 00000100:00100000:2.0:1767321213.158843:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6523 00000100:00100000:2.0:1767321213.158847:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985623040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321213.158855:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321213.158876:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985623040:12345-0@lo:38:kworker.0 Request processed in 28us (426us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321213.158882:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6523 00000100:00100000:2.0:1767321213.158891:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6524 00000100:00100000:2.0:1767321213.158894:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985623296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321213.158902:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321213.158921:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985623296:12345-0@lo:38:kworker.0 Request processed in 25us (435us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321213.158926:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6524 00000100:00080000:0.0:1767321213.159060:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321213.159066:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321213.159070:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985622656:0@lo:38:kworker.0 00000100:00080000:0.0:1767321213.159079:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321213.159082:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321213.159096:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985622784:0@lo:38:kworker.0 00000100:00080000:0.0:1767321213.159105:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321213.159108:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321213.159110:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985623040:0@lo:38:kworker.0 00000100:00080000:0.0:1767321213.159117:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321213.159120:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321213.159123:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985623296:0@lo:38:kworker.0 00010000:00010000:1.0:1767321213.787402:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321214.685051:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985623680:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321214.685156:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321214.685437:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321214.685465:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985623808:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321214.685534:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321214.685929:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985623680 00000100:00100000:0.0:1767321214.685941:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985623680 00000100:00100000:0.0:1767321214.686008:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985623808 00000100:00100000:0.0:1767321214.686012:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985623808 00000100:00100000:0.0:1767321214.686044:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2898 00000100:00100000:0.0:1767321214.686055:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985623680:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321214.686097:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321214.686108:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2899 00000100:00100000:2.0:1767321214.686127:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985623808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321214.686190:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321214.686253:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985623808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 134us (718us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321214.686261:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2899 00000100:00100000:1.0:1767321214.686370:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985623808:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321214.686388:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985623680:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321214.686627:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985623680:12345-0@lo:13:osp-pre-0-1.0 Request processed in 573us (1471us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321214.686636:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2898 00010000:00010000:2.0:1767321214.815367:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321215.835483:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321216.862057:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321217.243423:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321192/real 1767321192] req@ffff8ce188f04000 x1853163985618432/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321217 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321217.243457:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce188f04000 x1853163985618432/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321217 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321217.243471:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.243497:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321217.243501:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985618432:0@lo:250:kworker.0 00000100:00080000:0.0:1767321217.243586:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.243592:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321217.243595:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.243599:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6440 00000100:00080000:0.0:1767321217.243604:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.243610:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321217.243646:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7480 x1853163985623936/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321217.243671:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.243675:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.243677:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.243681:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6461 00000100:00080000:0.0:1767321217.243683:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.243687:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321217.243700:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7800 x1853163985624064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321217.243707:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.243710:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.243711:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.243714:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6461 00000100:00100000:2.0:1767321217.243810:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985623936:0@lo:250:kworker.0 00000400:00000100:2.0:1767321217.243854:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985623936 offset 0 length 520: 4 00000100:00100000:2.0:1767321217.243907:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985624064:0@lo:38:kworker.0 00000100:00100000:2.0:1767321217.243940:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321217.244020:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624064 00000100:00100000:3.0:1767321217.244028:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624064 00000100:00100000:3.0:1767321217.244072:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6525 00000100:00100000:3.0:1767321217.244079:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985624064:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321217.244102:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321217.244159:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985624064:12345-0@lo:38:kworker.0 Request processed in 80us (218us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321217.244187:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6525 00000100:00080000:1.0:1767321217.244279:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321217.244284:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321217.244287:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985624064:0@lo:38:kworker.0 00000100:00080000:0.0:1767321217.244347:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.244352:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321217.244369:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985624192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321217.244431:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985624320:0@lo:400:kworker.0 00000100:00100000:3.0:1767321217.244460:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321217.244500:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.244504:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.244507:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.244510:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6461 00000100:00080000:0.0:1767321217.244513:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.244516:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321217.244529:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6680 x1853163985624448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321217.244578:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985624576:0@lo:400:kworker.0 00000100:00100000:1.0:1767321217.244605:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321217.244637:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624320 00000100:00100000:1.0:1767321217.244641:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624320 00000100:00100000:1.0:1767321217.244678:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624576 00000100:00100000:1.0:1767321217.244682:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624576 00000100:00100000:1.0:1767321217.244703:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5689 00000100:00100000:1.0:1767321217.244710:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985624320:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321217.244872:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5690 00000100:00100000:2.0:1767321217.244882:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985624576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321217.244931:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985624576:12345-0@lo:400:kworker.0 Request processed in 51us (324us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321217.244937:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5690 00000100:00100000:2.0:1767321217.244967:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985624576:0@lo:400:kworker.0 00000100:00080000:2.0:1767321217.245001:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321217.245024:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985624192:0@lo:38:kworker.0 00000100:00100000:2.0:1767321217.245051:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321217.245070:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985624448:0@lo:38:kworker.0 00000100:00100000:2.0:1767321217.245089:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321217.245105:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624448 00000100:00100000:2.0:1767321217.245109:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624448 00000100:00100000:2.0:1767321217.245128:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6526 00000100:00100000:2.0:1767321217.245133:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985624448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321217.245146:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321217.245188:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985624448:12345-0@lo:38:kworker.0 Request processed in 53us (98us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321217.245194:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6526 00000100:00080000:2.0:1767321217.245217:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.245221:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321217.245224:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985624448:0@lo:38:kworker.0 00000100:00100000:2.0:1767321217.245255:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624192 00000100:00100000:2.0:1767321217.245258:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624192 00000100:00100000:2.0:1767321217.245277:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2122 00000100:00100000:2.0:1767321217.245281:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985624192:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321217.245291:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321217.245318:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985624192:12345-0@lo:38:kworker.0 Request processed in 34us (265us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321217.245323:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2122 00000100:00080000:2.0:1767321217.245341:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.245344:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321217.245347:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985624192:0@lo:38:kworker.0 00000100:00080000:0.0:1767321217.249771:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.249782:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.249787:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.249796:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6461 00000100:00080000:0.0:1767321217.249801:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.249808:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321217.249849:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7b80 x1853163985624704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321217.249968:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985624960:0@lo:400:kworker.0 00000100:00100000:0.0:1767321217.250038:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321217.250077:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321217.250083:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985624832:0@lo:400:kworker.0 00000100:00100000:0.0:1767321217.250104:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321217.250138:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624960 00000100:00100000:0.0:1767321217.250144:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624960 00000100:00100000:0.0:1767321217.250217:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624832 00000100:00100000:0.0:1767321217.250221:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624832 00000100:00100000:0.0:1767321217.250245:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3675 00000100:00100000:0.0:1767321217.250254:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985624960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321217.250304:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985624960:12345-0@lo:400:kworker.0 Request processed in 51us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321217.250313:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3675 00000100:00100000:0.0:1767321217.250330:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3676 00000100:00100000:0.0:1767321217.250336:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985624832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321217.250361:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985624832:12345-0@lo:400:kworker.0 Request processed in 26us (257us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321217.250367:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3676 00000100:00080000:0.0:1767321217.250390:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321217.250395:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321217.250423:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985624832:0@lo:400:kworker.0 00000100:00100000:0.0:1767321217.250434:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985624960:0@lo:400:kworker.0 00000100:00100000:2.0:1767321217.250643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985624704:0@lo:38:kworker.0 00000100:00100000:2.0:1767321217.250675:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321217.250827:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985624704 00000100:00100000:0.0:1767321217.250831:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985624704 00000100:00100000:0.0:1767321217.250872:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6527 00000100:00100000:0.0:1767321217.250877:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985624704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321217.250895:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321217.250927:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985624704:12345-0@lo:38:kworker.0 Request processed in 49us (252us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321217.250933:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6527 00000100:00100000:1.0:1767321217.251026:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985624320:12345-0@lo:400:kworker.0 Request processed in 6315us (6564us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321217.251033:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5689 00000100:00080000:2.0:1767321217.251213:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.251218:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321217.251222:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985624704:0@lo:38:kworker.0 00000100:00100000:3.0:1767321217.255007:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985624320:0@lo:400:kworker.0 00010000:00010000:1.0:1767321217.884246:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321218.219797:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321218.220022:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321218.221860:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660992 00000100:00100000:0.0:1767321218.221880:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660992 00000100:00100000:0.0:1767321218.222032:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920660864 00000100:00100000:0.0:1767321218.222036:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920660864 00000100:00100000:0.0:1767321218.222066:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5691 00000100:00100000:0.0:1767321218.222077:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920660992:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321218.222665:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920660992:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 587us (2880us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321218.222677:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5691 00000100:00100000:0.0:1767321218.222704:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6528 00000100:00100000:0.0:1767321218.222709:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920660864:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321218.222737:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321218.222771:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920660864:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 61us (2750us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321218.222778:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6528 00000400:00000100:3.0:1767321218.226537:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920661120 offset 0 length 520: 4 00000100:00080000:2.0:1767321218.226763:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321218.907986:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321219.803981:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321219.804196:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321219.804224:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985625216:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321219.804329:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321219.804766:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625216 00000100:00100000:0.0:1767321219.804778:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625216 00000100:00100000:0.0:1767321219.804875:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2900 00000100:00100000:0.0:1767321219.804885:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985625216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321219.804925:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321219.805001:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985625216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (671us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321219.805009:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2900 00000100:00100000:0.0:1767321219.805075:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985625216:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321219.805367:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321219.806095:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985625088:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321219.806707:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321219.806764:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625088 00000100:00100000:0.0:1767321219.806774:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625088 00000100:00100000:0.0:1767321219.806911:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2901 00000100:00100000:0.0:1767321219.806922:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985625088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321219.806967:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321219.807019:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985625088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (311us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321219.807027:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2901 00000100:00100000:3.0:1767321219.807206:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985625088:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321219.807399:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321219.931532:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321220.955550:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321221.982692:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321222.363351:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321222.363365:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321222.363369:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321222.363373:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321222.363380:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321222.363391:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6465 00000100:00080000:0.0:1767321222.363396:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321222.363406:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321222.363461:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7100 x1853163985625344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321222.363492:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321222.363496:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321222.363497:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321222.363502:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6465 00000100:00080000:0.0:1767321222.363504:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321222.363508:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321222.363524:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985625472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321222.363704:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985625600:0@lo:400:kworker.0 00000100:00100000:1.0:1767321222.363800:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321222.363954:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985625344:0@lo:38:kworker.0 00000100:00100000:2.0:1767321222.363989:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321222.364014:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985625472:0@lo:38:kworker.0 00000100:00100000:2.0:1767321222.364039:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321222.364106:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625472 00000100:00100000:2.0:1767321222.364115:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625472 00000100:00100000:2.0:1767321222.364157:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2123 00000100:00100000:2.0:1767321222.364183:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985625472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321222.364208:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321222.364255:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985625472:12345-0@lo:38:kworker.0 Request processed in 72us (214us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321222.364263:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2123 00000100:00080000:2.0:1767321222.364312:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321222.364316:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321222.364319:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985625472:0@lo:38:kworker.0 00000100:00080000:0.0:1767321222.367084:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321222.367106:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321222.367112:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321222.367121:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6465 00000100:00080000:0.0:1767321222.367127:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321222.367137:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321222.367223:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7b80 x1853163985625728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321222.367347:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985625856:0@lo:400:kworker.0 00000100:00100000:2.0:1767321222.367439:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321222.367508:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985625728:0@lo:38:kworker.0 00000100:00100000:2.0:1767321222.367534:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321222.367596:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625600 00000100:00100000:2.0:1767321222.367605:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625600 00000100:00100000:2.0:1767321222.367664:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625344 00000100:00100000:2.0:1767321222.367668:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625344 00000100:00100000:2.0:1767321222.367682:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625856 00000100:00100000:2.0:1767321222.367685:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625856 00000100:00100000:2.0:1767321222.367699:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625728 00000100:00100000:2.0:1767321222.367701:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625728 00000100:00100000:2.0:1767321222.367712:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5692 00000100:00100000:2.0:1767321222.367720:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985625600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321222.367885:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5693 00000100:00100000:3.0:1767321222.367898:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985625856:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321222.367947:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985625856:12345-0@lo:400:kworker.0 Request processed in 53us (507us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321222.367954:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5693 00000100:00100000:3.0:1767321222.367977:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6529 00000100:00100000:3.0:1767321222.367982:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985625344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321222.368004:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321222.368039:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985625344:12345-0@lo:38:kworker.0 Request processed in 56us (4049us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321222.368046:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6529 00000100:00100000:3.0:1767321222.368058:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6530 00000100:00100000:3.0:1767321222.368062:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985625728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321222.368071:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321222.368096:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985625728:12345-0@lo:38:kworker.0 Request processed in 33us (561us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321222.368103:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6530 00000100:00100000:3.0:1767321222.368331:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985625856:0@lo:400:kworker.0 00000100:00080000:3.0:1767321222.368371:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321222.368376:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321222.368380:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985625728:0@lo:38:kworker.0 00000100:00080000:3.0:1767321222.368390:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321222.368394:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321222.368397:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985625344:0@lo:38:kworker.0 00000100:00080000:3.0:1767321222.368562:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321222.372920:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321222.372936:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321222.372942:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321222.372952:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6465 00000100:00080000:0.0:1767321222.372958:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321222.372966:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321222.373022:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7100 x1853163985625984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321222.373104:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985625984:0@lo:38:kworker.0 00000100:00100000:3.0:1767321222.373471:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321222.373565:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985625984 00000100:00100000:3.0:1767321222.373576:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985625984 00000100:00100000:0.0:1767321222.373621:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985626240:0@lo:400:kworker.0 00000100:00100000:0.0:1767321222.373662:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321222.373685:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6531 00000100:00100000:0.0:1767321222.373693:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321222.373697:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985625984:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321222.373702:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985626112:0@lo:400:kworker.0 00000100:00100000:0.0:1767321222.373731:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767321222.373731:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321222.373762:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985626240 00000100:00100000:0.0:1767321222.373777:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985626240 00000100:00100000:3.0:1767321222.373802:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985625984:12345-0@lo:38:kworker.0 Request processed in 106us (331us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321222.373812:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6531 00000100:00100000:0.0:1767321222.373831:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985626112 00000100:00100000:0.0:1767321222.373835:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985626112 00000100:00080000:3.0:1767321222.373868:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321222.374701:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321222.374708:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985625984:0@lo:38:kworker.0 00000100:00100000:3.0:1767321222.374806:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3677 00000100:00100000:3.0:1767321222.374820:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985626240:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321222.374951:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985626240:12345-0@lo:400:kworker.0 Request processed in 136us (1287us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321222.374960:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3677 00000100:00080000:3.0:1767321222.375007:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321222.376435:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3678 00000100:00100000:0.0:1767321222.376447:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985626112:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321222.376811:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985626112:12345-0@lo:400:kworker.0 Request processed in 365us (3080us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321222.376821:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3678 00000100:00100000:0.0:1767321222.376895:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985626112:0@lo:400:kworker.0 00000100:00100000:0.0:1767321222.376915:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985626240:0@lo:400:kworker.0 00000100:00080000:3.0:1767321222.376945:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321222.377273:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985625600:12345-0@lo:400:kworker.0 Request processed in 9550us (13471us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321222.377284:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5692 00000100:00100000:1.0:1767321222.381867:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985625600:0@lo:400:kworker.0 00010000:00010000:0.0:1767321223.003344:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321223.340775:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321223.340902:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920661376 00000100:00100000:0.0:1767321223.340916:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920661376 00000100:00100000:0.0:1767321223.341020:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5694 00000100:00100000:0.0:1767321223.341033:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920661376:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321223.341154:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920661376:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (403us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321223.341200:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5694 00000100:00080000:3.0:1767321223.341931:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321223.342028:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321223.342280:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920661248 00000100:00100000:2.0:1767321223.342291:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920661248 00000100:00100000:2.0:1767321223.342330:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6532 00000100:00100000:2.0:1767321223.342339:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920661248:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321223.342366:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321223.342435:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920661248:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 96us (409us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321223.342442:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6532 00010000:00010000:1.0:1767321224.027451:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321224.923597:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321224.923629:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985626368:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321224.923739:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321224.923962:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985626368 00000100:00100000:1.0:1767321224.923973:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985626368 00000100:00100000:1.0:1767321224.924052:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2902 00000100:00100000:1.0:1767321224.924064:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985626368:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321224.924105:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321224.924122:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985626496:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321224.924214:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985626368:12345-0@lo:13:osp-pre-1-1.0 Request processed in 151us (473us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321224.924223:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2902 00000100:00080000:3.0:1767321224.924465:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321224.926889:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321224.927031:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f04000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985626368:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321224.927194:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985626496 00000100:00100000:1.0:1767321224.927204:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985626496 00000100:00100000:1.0:1767321224.927349:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2903 00000100:00100000:1.0:1767321224.927361:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985626496:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321224.927401:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321224.927510:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985626496:12345-0@lo:13:osp-pre-0-1.0 Request processed in 151us (633us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321224.927519:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2903 00000100:00100000:0.0:1767321224.929531:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985626496:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321224.930995:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321225.051751:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321226.075473:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321227.099467:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321227.483355:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321227.483370:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321227.483374:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321227.483379:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321227.483385:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321227.483396:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6470 00000100:00080000:0.0:1767321227.483402:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321227.483411:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321227.484249:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3b80 x1853163985626624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321227.484366:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321227.484372:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321227.484375:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321227.484379:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6470 00000100:00080000:0.0:1767321227.484382:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321227.484387:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321227.484415:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985626752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321227.484442:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985626624:0@lo:38:kworker.0 00000100:00100000:3.0:1767321227.484554:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321227.484624:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985626752:0@lo:38:kworker.0 00000100:00100000:3.0:1767321227.484655:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321227.484697:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985626880:0@lo:400:kworker.0 00000100:00100000:3.0:1767321227.484720:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985626752 00000100:00100000:2.0:1767321227.484730:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321227.484730:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985626752 00000100:00100000:2.0:1767321227.484786:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985626624 00000100:00100000:3.0:1767321227.484786:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2124 00000100:00100000:2.0:1767321227.484789:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985626624 00000100:00100000:3.0:1767321227.484795:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985626752:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321227.484814:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985626880 00010000:00020000:3.0:1767321227.484824:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321227.484826:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985626880 00000100:00100000:2.0:1767321227.484870:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5695 00000100:00100000:3.0:1767321227.484872:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985626752:12345-0@lo:38:kworker.0 Request processed in 78us (216us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321227.484878:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985626880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321227.484879:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2124 00000100:00080000:3.0:1767321227.484912:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321227.484926:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985626880:12345-0@lo:400:kworker.0 Request processed in 50us (194us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321227.484927:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6533 00000100:00100000:3.0:1767321227.484931:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985626624:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321227.484932:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5695 00010000:00020000:3.0:1767321227.484941:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321227.484980:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321227.484984:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321227.484987:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321227.484992:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6470 00000100:00080000:0.0:1767321227.484994:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321227.484999:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321227.485016:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdbb80 x1853163985627008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321227.485035:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321227.485038:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321227.485040:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321227.485043:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6470 00000100:00080000:0.0:1767321227.485045:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321227.485048:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321227.485057:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bda680 x1853163985627264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321227.485107:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985627136:0@lo:400:kworker.0 00000100:00100000:1.0:1767321227.485202:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985627392:0@lo:400:kworker.0 00000100:00100000:2.0:1767321227.485319:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985626880:0@lo:400:kworker.0 00000100:00100000:3.0:1767321227.485347:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985626624:12345-0@lo:38:kworker.0 Request processed in 413us (793us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321227.485354:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6533 00000100:00100000:2.0:1767321227.485357:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985627008:0@lo:38:kworker.0 00000100:00100000:3.0:1767321227.485382:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985627520:0@lo:400:kworker.0 00000100:00100000:2.0:1767321227.485384:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321227.485401:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985627264:0@lo:38:kworker.0 00000100:00100000:3.0:1767321227.485409:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321227.485422:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321227.485439:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627008 00000100:00100000:3.0:1767321227.485442:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627008 00000100:00080000:2.0:1767321227.485464:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321227.485476:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321227.485480:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985626624:0@lo:38:kworker.0 00000100:00080000:2.0:1767321227.485491:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321227.485495:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321227.485498:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985626752:0@lo:38:kworker.0 00000100:00100000:2.0:1767321227.485661:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627520 00000100:00100000:2.0:1767321227.485665:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627520 00000100:00100000:2.0:1767321227.485720:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3679 00000100:00100000:2.0:1767321227.485728:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985627520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321227.485771:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985627520:12345-0@lo:400:kworker.0 Request processed in 44us (361us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321227.485778:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3679 00000100:00080000:2.0:1767321227.485800:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321227.485835:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985627520:0@lo:400:kworker.0 00000100:00100000:2.0:1767321227.485872:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627264 00000100:00100000:2.0:1767321227.485876:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627264 00000100:00100000:0.0:1767321227.486445:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321227.487069:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627136 00000100:00100000:2.0:1767321227.487078:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627136 00000100:00100000:2.0:1767321227.487130:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5696 00000100:00100000:2.0:1767321227.487139:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985627136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321227.487214:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985627136:12345-0@lo:400:kworker.0 Request processed in 75us (771us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321227.487222:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5696 00000100:00100000:2.0:1767321227.487241:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6534 00000100:00100000:2.0:1767321227.487246:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985627008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321227.487267:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321227.487318:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985627008:12345-0@lo:38:kworker.0 Request processed in 70us (1933us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321227.487323:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6534 00000100:00100000:2.0:1767321227.487337:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6535 00000100:00100000:2.0:1767321227.487341:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985627264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321227.487350:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321227.487374:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985627264:12345-0@lo:38:kworker.0 Request processed in 32us (1951us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321227.487381:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6535 00000100:00080000:2.0:1767321227.487403:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321227.487433:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321227.487438:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321227.487442:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985627008:0@lo:38:kworker.0 00000100:00080000:2.0:1767321227.487453:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321227.487456:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321227.487460:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985627264:0@lo:38:kworker.0 00000100:00100000:2.0:1767321227.487909:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985627136:0@lo:400:kworker.0 00000100:00100000:1.0:1767321227.488000:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321227.488112:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627392 00000100:00100000:2.0:1767321227.488118:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627392 00000100:00100000:2.0:1767321227.488240:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3680 00000100:00100000:2.0:1767321227.488249:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985627392:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321227.488294:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985627392:12345-0@lo:400:kworker.0 Request processed in 46us (294us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321227.488300:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3680 00000100:00100000:1.0:1767321227.488375:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985627392:0@lo:400:kworker.0 00000100:00080000:3.0:1767321227.488420:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321228.123773:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321228.463309:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321228.463446:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920661632 00000100:00100000:0.0:1767321228.463478:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920661632 00000100:00100000:0.0:1767321228.463603:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5697 00000100:00100000:0.0:1767321228.463617:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920661632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321228.463736:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920661632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 121us (690us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321228.463745:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5697 00000100:00080000:3.0:1767321228.464080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321228.465929:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321228.466007:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920661504 00000100:00100000:1.0:1767321228.466018:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920661504 00000100:00100000:1.0:1767321228.466066:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6536 00000100:00100000:1.0:1767321228.466076:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920661504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321228.466108:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321228.466205:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920661504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 130us (287us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321228.466212:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6536 00010000:00010000:0.0:1767321229.149206:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321230.043817:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985627648:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321230.043950:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985627776:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321230.044132:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321230.044445:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627776 00000100:00100000:0.0:1767321230.044457:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627776 00000100:00100000:0.0:1767321230.044538:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2904 00000100:00100000:0.0:1767321230.044550:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985627776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321230.044590:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321230.044656:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985627776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (523us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321230.044664:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2904 00000100:00100000:2.0:1767321230.045713:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321230.045953:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627648 00000100:00100000:0.0:1767321230.045963:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627648 00000100:00100000:0.0:1767321230.046024:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2905 00000100:00100000:0.0:1767321230.046035:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985627648:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321230.046063:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321230.046134:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985627648:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (424us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321230.046141:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2905 00000100:00100000:3.0:1767321230.046496:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985627648:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321230.046579:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321230.046584:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321230.046618:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985627776:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321230.171815:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321231.195507:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321232.221762:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321232.604222:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321232.604237:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321232.604241:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321232.604247:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321232.604252:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321232.604263:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6475 00000100:00080000:2.0:1767321232.604270:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321232.604279:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321232.604334:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffdf80 x1853163985627904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321232.604372:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321232.604376:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321232.604378:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321232.604382:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6475 00000100:00080000:2.0:1767321232.604384:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321232.604388:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321232.604406:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985628032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321232.604453:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321232.604457:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321232.604459:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321232.604463:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6475 00000100:00080000:2.0:1767321232.604465:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321232.604468:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321232.604481:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13800 x1853163985628288/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321232.604498:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321232.604502:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321232.604504:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321232.604507:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6475 00000100:00080000:2.0:1767321232.604509:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321232.604513:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321232.604523:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13b80 x1853163985628544/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321232.604592:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985628416:0@lo:400:kworker.0 00000100:00100000:2.0:1767321232.604684:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321232.605255:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628416 00000100:00100000:1.0:1767321232.605266:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628416 00000100:00100000:1.0:1767321232.605353:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5698 00000100:00100000:1.0:1767321232.605366:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985628416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.605484:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985627904:0@lo:38:kworker.0 00000100:00100000:3.0:1767321232.605521:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321232.605590:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985628032:0@lo:38:kworker.0 00000100:00100000:3.0:1767321232.605619:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321232.605645:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985628288:0@lo:38:kworker.0 00000100:00100000:3.0:1767321232.605667:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321232.605685:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985628544:0@lo:38:kworker.0 00000100:00100000:3.0:1767321232.605707:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321232.605761:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985627904 00000100:00100000:3.0:1767321232.605765:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985627904 00000100:00100000:3.0:1767321232.605787:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628288 00000100:00100000:3.0:1767321232.605790:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628288 00000100:00100000:3.0:1767321232.605796:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628544 00000100:00100000:3.0:1767321232.605798:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628544 00000100:00100000:3.0:1767321232.605809:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6537 00000100:00100000:3.0:1767321232.605814:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985627904:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321232.605837:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321232.605873:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985627904:12345-0@lo:38:kworker.0 Request processed in 57us (351us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321232.605881:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6537 00000100:00100000:3.0:1767321232.605901:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6538 00000100:00100000:3.0:1767321232.605905:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985628288:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321232.605913:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321232.605935:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985628288:12345-0@lo:38:kworker.0 Request processed in 28us (267us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321232.605940:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6538 00000100:00100000:3.0:1767321232.605949:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6539 00000100:00100000:3.0:1767321232.605953:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985628544:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321232.605962:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321232.605981:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985628544:12345-0@lo:38:kworker.0 Request processed in 25us (272us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321232.605986:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6539 00000100:00100000:3.0:1767321232.606004:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628032 00000100:00100000:3.0:1767321232.606008:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628032 00000100:00100000:3.0:1767321232.606029:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2125 00000100:00100000:3.0:1767321232.606035:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985628032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321232.606044:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321232.606083:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985628032:12345-0@lo:38:kworker.0 Request processed in 48us (463us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321232.606090:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2125 00000100:00080000:3.0:1767321232.606149:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321232.606155:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321232.606160:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985627904:0@lo:38:kworker.0 00000100:00080000:3.0:1767321232.606223:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321232.606227:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321232.606230:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985628032:0@lo:38:kworker.0 00000100:00080000:3.0:1767321232.606237:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321232.606240:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321232.606243:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985628288:0@lo:38:kworker.0 00000100:00080000:3.0:1767321232.606250:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321232.606253:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321232.606256:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985628544:0@lo:38:kworker.0 00000100:00100000:3.0:1767321232.606294:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985628672:0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606329:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321232.606360:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321232.606368:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985628800:0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606392:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321232.606438:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628672 00000100:00100000:3.0:1767321232.606442:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628672 00000100:00100000:3.0:1767321232.606478:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628800 00000100:00100000:3.0:1767321232.606481:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628800 00000100:00100000:3.0:1767321232.606502:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3681 00000100:00100000:3.0:1767321232.606509:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985628672:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606550:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985628672:12345-0@lo:400:kworker.0 Request processed in 42us (220us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321232.606557:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3681 00000100:00100000:3.0:1767321232.606569:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3682 00000100:00100000:3.0:1767321232.606575:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985628800:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606615:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985628800:12345-0@lo:400:kworker.0 Request processed in 40us (221us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321232.606621:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3682 00000100:00100000:3.0:1767321232.606654:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985628800:0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606663:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985628672:0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606697:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985628160:0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606726:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321232.606748:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628160 00000100:00100000:3.0:1767321232.606752:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628160 00000100:00100000:3.0:1767321232.606781:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5699 00000100:00100000:3.0:1767321232.606787:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985628160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321232.606817:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985628160:12345-0@lo:400:kworker.0 Request processed in 30us (89us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321232.606822:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5699 00000100:00100000:3.0:1767321232.606850:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985628160:0@lo:400:kworker.0 00000100:00080000:0.0:1767321232.608305:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321232.608317:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321232.608321:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321232.619555:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985628416:12345-0@lo:400:kworker.0 Request processed in 14175us (14852us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321232.619575:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5698 00000100:00100000:2.0:1767321232.619844:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985628416:0@lo:400:kworker.0 00010000:00010000:2.0:1767321233.256716:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321233.598968:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321233.599038:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321233.599199:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920661888 00000100:00100000:1.0:1767321233.599213:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920661888 00000100:00100000:1.0:1767321233.599315:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920661760 00000100:00100000:1.0:1767321233.599320:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920661760 00000100:00100000:1.0:1767321233.599341:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5700 00000100:00100000:1.0:1767321233.599353:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920661888:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321233.599456:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920661888:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 105us (510us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321233.599477:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5700 00000100:00100000:1.0:1767321233.599655:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6540 00000100:00100000:1.0:1767321233.599661:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920661760:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321233.599687:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321233.599722:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920661760:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 60us (684us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321233.599730:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6540 00000100:00080000:2.0:1767321233.601143:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321234.267402:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321235.163417:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985628928:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321235.163485:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985629056:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321235.163491:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321235.163540:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321235.163619:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985628928 00000100:00100000:0.0:1767321235.163627:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985628928 00000100:00100000:0.0:1767321235.163686:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629056 00000100:00100000:0.0:1767321235.163690:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629056 00000100:00100000:0.0:1767321235.163711:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2906 00000100:00100000:0.0:1767321235.163718:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985628928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321235.163748:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767321235.165638:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321235.165648:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321235.165674:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2907 00000100:00100000:2.0:1767321235.165694:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985629056:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321235.165724:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321235.165799:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985629056:12345-0@lo:13:osp-pre-1-1.0 Request processed in 116us (2268us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321235.165806:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2907 00000100:00100000:2.0:1767321235.165859:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985629056:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321235.175384:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985628928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 11645us (11871us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321235.175399:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2906 00000100:00100000:3.0:1767321235.177247:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985628928:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767321235.293971:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321236.315839:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321237.343084:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321237.723805:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321237.723815:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321237.723818:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321237.723822:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321237.723826:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321237.723835:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6480 00000100:00080000:3.0:1767321237.723839:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321237.723845:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321237.723884:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdbb80 x1853163985629184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321237.723899:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321237.723902:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321237.723903:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321237.723907:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6480 00000100:00080000:3.0:1767321237.723911:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321237.723914:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321237.723926:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb100 x1853163985629312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321237.723968:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321237.723971:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321237.723973:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321237.723977:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6480 00000100:00080000:3.0:1767321237.723980:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321237.723983:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321237.723997:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc3b80 x1853163985629568/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321237.724023:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321237.724026:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321237.724027:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321237.724031:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6480 00000100:00080000:3.0:1767321237.724033:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321237.724036:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321237.724047:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ffdf80 x1853163985629824/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321237.724099:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985629952:0@lo:400:kworker.0 00000100:00100000:3.0:1767321237.724160:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321237.724237:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321237.724244:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985630080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321237.724270:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321237.724366:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985629184:0@lo:38:kworker.0 00000100:00100000:0.0:1767321237.724398:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321237.724431:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985629312:0@lo:38:kworker.0 00000100:00100000:0.0:1767321237.724465:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321237.724497:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985629568:0@lo:38:kworker.0 00000100:00100000:0.0:1767321237.724517:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321237.724532:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985629824:0@lo:38:kworker.0 00000100:00100000:0.0:1767321237.724551:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321237.724589:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629952 00000100:00100000:0.0:1767321237.724597:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629952 00000100:00100000:0.0:1767321237.724641:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630080 00000100:00100000:0.0:1767321237.724645:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630080 00000100:00100000:0.0:1767321237.724665:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3683 00000100:00100000:0.0:1767321237.724673:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985629952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321237.724720:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985629952:12345-0@lo:400:kworker.0 Request processed in 47us (559us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321237.724727:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3683 00000100:00100000:0.0:1767321237.724742:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3684 00000100:00100000:0.0:1767321237.724748:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985630080:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321237.724773:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985630080:12345-0@lo:400:kworker.0 Request processed in 25us (502us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321237.724779:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3684 00000100:00100000:0.0:1767321237.724800:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629184 00000100:00100000:0.0:1767321237.724804:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629184 00000100:00100000:0.0:1767321237.724821:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629568 00000100:00100000:0.0:1767321237.724824:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629568 00000100:00100000:0.0:1767321237.724834:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629824 00000100:00100000:0.0:1767321237.724837:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629824 00000100:00100000:0.0:1767321237.724849:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6541 00000100:00100000:0.0:1767321237.724854:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985629184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321237.724872:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321237.724904:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985629184:12345-0@lo:38:kworker.0 Request processed in 49us (506us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321237.724911:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6541 00000100:00100000:0.0:1767321237.724923:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6542 00000100:00100000:0.0:1767321237.724926:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985629568:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321237.724935:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321237.724958:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985629568:12345-0@lo:38:kworker.0 Request processed in 30us (440us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321237.724963:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6542 00000100:00100000:0.0:1767321237.724973:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6543 00000100:00100000:0.0:1767321237.724977:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985629824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321237.724985:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321237.725005:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985629824:12345-0@lo:38:kworker.0 Request processed in 27us (454us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321237.725010:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6543 00000100:00080000:0.0:1767321237.725040:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321237.725045:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321237.725049:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985629184:0@lo:38:kworker.0 00000100:00080000:0.0:1767321237.725058:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321237.725061:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321237.725064:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985629568:0@lo:38:kworker.0 00000100:00080000:0.0:1767321237.725070:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321237.725073:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321237.725076:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985629824:0@lo:38:kworker.0 00000100:00100000:2.0:1767321237.725192:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985629696:0@lo:400:kworker.0 00000100:00100000:2.0:1767321237.725223:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321237.725240:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321237.725246:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985629440:0@lo:400:kworker.0 00000100:00100000:2.0:1767321237.725266:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321237.725283:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321237.725287:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321237.725342:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629696 00000100:00100000:0.0:1767321237.725347:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629696 00000100:00100000:0.0:1767321237.725378:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629440 00000100:00100000:0.0:1767321237.725381:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629440 00000100:00100000:0.0:1767321237.725396:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5701 00000100:00100000:0.0:1767321237.725402:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985629696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321237.725434:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985629696:12345-0@lo:400:kworker.0 Request processed in 32us (210us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321237.725440:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5701 00000100:00100000:0.0:1767321237.725452:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5702 00000100:00100000:0.0:1767321237.725459:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985629440:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321237.725497:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985629440:12345-0@lo:400:kworker.0 Request processed in 25us (217us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321237.725503:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5702 00000100:00080000:2.0:1767321237.725732:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321237.725754:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985629440:0@lo:400:kworker.0 00000100:00100000:2.0:1767321237.725763:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985629696:0@lo:400:kworker.0 00000100:00100000:3.0:1767321237.728910:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985629312 00000100:00100000:3.0:1767321237.728927:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985629312 00000100:00100000:3.0:1767321237.728978:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2126 00000100:00100000:3.0:1767321237.728987:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985629312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321237.729290:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321237.729551:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985629312:12345-0@lo:38:kworker.0 Request processed in 563us (5083us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321237.729560:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2126 00000100:00100000:3.0:1767321237.729637:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985630080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321237.729651:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985629952:0@lo:400:kworker.0 00000100:00080000:1.0:1767321237.729804:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321237.729810:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321237.729813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985629312:0@lo:38:kworker.0 00010000:00010000:1.0:1767321238.363466:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321238.698108:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321238.698188:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920662144 00000100:00100000:3.0:1767321238.698198:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920662144 00000100:00100000:3.0:1767321238.698273:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5703 00000100:00100000:3.0:1767321238.698283:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920662144:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321238.698371:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920662144:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 89us (271us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321238.698379:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5703 00000100:00080000:2.0:1767321238.698445:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321238.700859:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321238.700943:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920662016 00000100:00100000:3.0:1767321238.700952:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920662016 00000100:00100000:3.0:1767321238.700995:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6544 00000100:00100000:3.0:1767321238.701003:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920662016:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321238.701025:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321238.701114:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920662016:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 110us (264us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321238.701122:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6544 00010000:00010000:2.0:1767321239.387373:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321240.283530:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985630208:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321240.283600:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321240.283742:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630208 00000100:00100000:0.0:1767321240.283750:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630208 00000100:00100000:0.0:1767321240.283813:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2908 00000100:00100000:0.0:1767321240.283822:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985630208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321240.283850:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321240.283899:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985630208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321240.283907:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2908 00000100:00100000:0.0:1767321240.283951:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985630208:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321240.284082:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321240.287535:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985630336:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321240.287611:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321240.287739:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630336 00000100:00100000:1.0:1767321240.287749:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630336 00000100:00100000:1.0:1767321240.287815:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2909 00000100:00100000:1.0:1767321240.287823:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985630336:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321240.287853:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321240.287891:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985630336:12345-0@lo:13:osp-pre-0-1.0 Request processed in 69us (280us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321240.287898:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2909 00000100:00080000:3.0:1767321240.288021:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321240.288054:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985630336:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321240.411365:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321241.435446:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321242.459533:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767321242.802007:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920662272 offset 0 length 520: 4 00000100:00100000:1.0:1767321242.802067:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321242.802304:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920662400 00000100:00100000:0.0:1767321242.802315:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920662400 00000100:00100000:0.0:1767321242.802363:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6545 00000100:00100000:0.0:1767321242.802371:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920662400:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321242.802405:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321242.802479:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920662400:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 107us (411us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321242.802487:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6545 00000100:00100000:2.0:1767321242.809010:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321242.809108:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920662528 00000100:00100000:0.0:1767321242.809117:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920662528 00000100:00100000:0.0:1767321242.809229:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5704 00000100:00100000:0.0:1767321242.809240:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920662528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767321242.809298:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321242.809326:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920662528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 88us (325us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321242.809334:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5704 00000100:00100000:0.0:1767321242.843295:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321217/real 1767321217] req@ffff8ce1a61f7480 x1853163985623936/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321242 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321242.843295:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.843302:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321242.843307:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.843312:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.843315:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.843321:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6485 00000100:00080000:3.0:1767321242.843326:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767321242.843330:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61f7480 x1853163985623936/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321242 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321242.843332:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00080000:0.0:1767321242.843353:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321242.843360:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767321242.843364:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985623936:0@lo:250:kworker.0 00000100:00100000:3.0:1767321242.843382:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985630464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321242.843406:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.843409:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.843410:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.843414:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6485 00000100:00100000:0.0:1767321242.843415:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985630464:0@lo:38:kworker.0 00000100:00080000:3.0:1767321242.843416:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321242.843419:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321242.843434:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3800 x1853163985630592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321242.843473:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321242.843475:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.843478:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.843480:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.843483:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6485 00000100:00080000:3.0:1767321242.843485:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321242.843488:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321242.843498:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c4e00 x1853163985630848/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321242.843514:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.843517:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.843519:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.843522:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6485 00000100:00080000:3.0:1767321242.843524:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767321242.843527:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985630592:0@lo:38:kworker.0 00000100:00080000:3.0:1767321242.843527:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321242.843536:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1c00 x1853163985631104/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321242.843549:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321242.843570:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985630848:0@lo:38:kworker.0 00000100:00080000:3.0:1767321242.843578:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.843581:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321242.843583:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.843586:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6465 00000100:00100000:0.0:1767321242.843588:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321242.843588:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321242.843592:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321242.843600:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0a80 x1853163985631488/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321242.843604:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985631104:0@lo:38:kworker.0 00000100:00100000:0.0:1767321242.843621:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321242.843627:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985630976:0@lo:400:kworker.0 00000100:00100000:0.0:1767321242.843637:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985631488:0@lo:250:kworker.0 00000400:00000100:0.0:1767321242.843650:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985631488 offset 0 length 520: 4 00000100:00100000:3.0:1767321242.843652:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321242.843669:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321242.843674:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985630720:0@lo:400:kworker.0 00000100:00100000:0.0:1767321242.843680:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630592 00000100:00100000:0.0:1767321242.843688:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630592 00000100:00100000:3.0:1767321242.843705:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321242.843721:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2127 00000100:00100000:3.0:1767321242.843724:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985631360:0@lo:400:kworker.0 00000100:00100000:0.0:1767321242.843728:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985630592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321242.843747:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321242.843747:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321242.843763:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321242.843767:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985631232:0@lo:400:kworker.0 00000100:00100000:0.0:1767321242.843782:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985630592:12345-0@lo:38:kworker.0 Request processed in 54us (231us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321242.843784:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321242.843789:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2127 00000100:00100000:3.0:1767321242.843805:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630464 00000100:00100000:3.0:1767321242.843809:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630464 00000100:00080000:0.0:1767321242.843820:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321242.843824:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321242.843827:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985630592:0@lo:38:kworker.0 00000100:00100000:3.0:1767321242.843829:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630848 00000100:00100000:3.0:1767321242.843831:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630848 00000100:00100000:3.0:1767321242.843838:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985631104 00000100:00100000:3.0:1767321242.843841:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985631104 00000100:00100000:0.0:1767321242.843845:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630976 00000100:00100000:0.0:1767321242.843848:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630976 00000100:00100000:3.0:1767321242.843848:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985630720 00000100:00100000:3.0:1767321242.843850:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985630720 00000100:00100000:1.0:1767321242.843921:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6546 00000100:00100000:1.0:1767321242.843926:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985630464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321242.843937:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321242.843972:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985630464:12345-0@lo:38:kworker.0 Request processed in 44us (499us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321242.843977:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6546 00000100:00100000:1.0:1767321242.843992:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6547 00000100:00100000:1.0:1767321242.843995:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985630848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321242.844003:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321242.844024:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985630848:12345-0@lo:38:kworker.0 Request processed in 27us (435us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321242.844029:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6547 00000100:00100000:1.0:1767321242.844038:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6548 00000100:00100000:1.0:1767321242.844042:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985631104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321242.844049:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321242.844066:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985631104:12345-0@lo:38:kworker.0 Request processed in 23us (444us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321242.844071:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6548 00000100:00080000:1.0:1767321242.844097:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.844100:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321242.844103:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985631104:0@lo:38:kworker.0 00000100:00080000:1.0:1767321242.844111:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.844114:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321242.844116:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985630848:0@lo:38:kworker.0 00000100:00080000:1.0:1767321242.844124:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.844126:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321242.844129:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985630464:0@lo:38:kworker.0 00000100:00100000:1.0:1767321242.844210:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985631360 00000100:00100000:1.0:1767321242.844214:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985631360 00000100:00100000:1.0:1767321242.844245:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985631232 00000100:00100000:1.0:1767321242.844247:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985631232 00000100:00100000:1.0:1767321242.844266:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3685 00000100:00100000:1.0:1767321242.844272:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985631360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321242.844313:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985631360:12345-0@lo:400:kworker.0 Request processed in 41us (564us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321242.844318:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3685 00000100:00100000:1.0:1767321242.844331:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3686 00000100:00100000:1.0:1767321242.844337:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985631232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321242.844360:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985631232:12345-0@lo:400:kworker.0 Request processed in 24us (575us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321242.844365:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3686 00000100:00080000:1.0:1767321242.844426:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321242.844430:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321242.844432:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321242.844515:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985631232:0@lo:400:kworker.0 00000100:00100000:2.0:1767321242.844525:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985631360:0@lo:400:kworker.0 00000100:00100000:3.0:1767321242.844857:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5705 00000100:00100000:0.0:1767321242.847622:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5706 00000100:00100000:0.0:1767321242.847638:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985630976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321242.847734:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985630976:12345-0@lo:400:kworker.0 Request processed in 100us (4079us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321242.847741:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5706 00000100:00100000:1.0:1767321242.851351:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985630976:0@lo:400:kworker.0 00000100:00100000:3.0:1767321242.854818:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985630720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321242.854930:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985630720:12345-0@lo:400:kworker.0 Request processed in 10066us (11222us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321242.854940:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5705 00000100:00100000:1.0:1767321242.855051:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985630720:0@lo:400:kworker.0 00010000:00010000:2.0:1767321243.492150:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321244.509413:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321245.404638:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985631616:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321245.404726:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321245.404787:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321245.404795:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985631744:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321245.404820:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321245.404940:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985631616 00000100:00100000:3.0:1767321245.404950:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985631616 00000100:00100000:3.0:1767321245.405011:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985631744 00000100:00100000:3.0:1767321245.405014:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985631744 00000100:00100000:3.0:1767321245.405034:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2910 00000100:00100000:3.0:1767321245.405043:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985631616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321245.405076:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321245.405119:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985631616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (393us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321245.405132:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2910 00000100:00100000:3.0:1767321245.405149:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2911 00000100:00100000:3.0:1767321245.405155:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985631744:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321245.405188:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321245.405215:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985631744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (394us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321245.405221:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2911 00000100:00080000:2.0:1767321245.405277:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321245.405282:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321245.405326:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985631744:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321245.405353:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985631616:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321245.537676:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321246.555650:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321247.579715:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321247.919095:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321247.919217:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321247.919332:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920662656 00000100:00100000:3.0:1767321247.919343:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920662656 00000100:00100000:3.0:1767321247.919386:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920662784 00000100:00100000:3.0:1767321247.919389:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920662784 00000100:00100000:3.0:1767321247.919440:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5707 00000100:00100000:3.0:1767321247.919451:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920662784:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767321247.919489:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321247.919534:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6549 00000100:00100000:1.0:1767321247.919545:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920662656:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321247.919577:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321247.919675:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920662656:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 130us (594us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321247.919684:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6549 00000100:00100000:3.0:1767321247.920379:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920662784:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 924us (1155us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321247.920391:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5707 00000100:00080000:3.0:1767321247.963847:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321247.963858:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321247.963862:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321247.963866:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321247.963873:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321247.963882:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6491 00000100:00080000:3.0:1767321247.963888:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321247.963895:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321247.963946:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4700 x1853163985631872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321247.963990:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321247.963993:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321247.963995:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321247.963999:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6491 00000100:00080000:3.0:1767321247.964002:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321247.964005:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321247.964018:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c6300 x1853163985632000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321247.964107:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985632128:0@lo:400:kworker.0 00000100:00100000:2.0:1767321247.964218:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321247.964399:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632128 00000100:00100000:1.0:1767321247.964408:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632128 00000100:00100000:1.0:1767321247.964467:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5708 00000100:00100000:1.0:1767321247.964476:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985632128:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321247.964552:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321247.964557:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321247.964559:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321247.964563:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6491 00000100:00080000:3.0:1767321247.964566:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321247.964570:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321247.964584:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c7100 x1853163985632256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321247.965025:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985631872:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.965069:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.965103:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985632000:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.965127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.965160:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985632256:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.965203:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.965246:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632000 00000100:00100000:0.0:1767321247.965252:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632000 00000100:00100000:0.0:1767321247.965277:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2128 00000100:00100000:0.0:1767321247.965282:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985632000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321247.965313:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321247.965349:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985632000:12345-0@lo:38:kworker.0 Request processed in 65us (221us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321247.965356:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2128 00000100:00100000:0.0:1767321247.965383:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985631872 00000100:00100000:0.0:1767321247.965387:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985631872 00000100:00100000:0.0:1767321247.965405:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632256 00000100:00100000:0.0:1767321247.965408:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632256 00000100:00100000:0.0:1767321247.965422:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6550 00000100:00100000:0.0:1767321247.965442:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985631872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321247.965451:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321247.965490:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985631872:12345-0@lo:38:kworker.0 Request processed in 61us (421us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321247.965496:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6550 00000100:00100000:0.0:1767321247.965507:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6551 00000100:00100000:0.0:1767321247.965511:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985632256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321247.965520:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321247.965540:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985632256:12345-0@lo:38:kworker.0 Request processed in 28us (336us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321247.965545:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6551 00000100:00080000:0.0:1767321247.965586:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321247.965591:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321247.965594:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985631872:0@lo:38:kworker.0 00000100:00080000:0.0:1767321247.965603:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321247.965606:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321247.965609:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985632000:0@lo:38:kworker.0 00000100:00080000:0.0:1767321247.965616:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321247.965619:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321247.965622:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985632256:0@lo:38:kworker.0 00000100:00080000:0.0:1767321247.965747:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321247.965776:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985632384:0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.965802:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.965828:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632384 00000100:00100000:0.0:1767321247.965831:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632384 00000100:00100000:0.0:1767321247.965860:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5709 00000100:00100000:0.0:1767321247.965865:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985632384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.965900:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985632384:12345-0@lo:400:kworker.0 Request processed in 34us (97us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321247.965906:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5709 00000100:00100000:0.0:1767321247.965932:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985632384:0@lo:400:kworker.0 00000100:00080000:0.0:1767321247.965946:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321247.966889:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985632128:12345-0@lo:400:kworker.0 Request processed in 2410us (2668us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321247.966899:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5708 00000100:00080000:3.0:1767321247.966988:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321247.966993:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321247.966996:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321247.967001:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6491 00000100:00080000:3.0:1767321247.967004:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321247.967009:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321247.967036:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c5f80 x1853163985632512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321247.967116:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985632768:0@lo:400:kworker.0 00000100:00100000:3.0:1767321247.967185:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321247.967209:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321247.967215:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985632640:0@lo:400:kworker.0 00000100:00100000:1.0:1767321247.967651:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632768 00000100:00100000:1.0:1767321247.967659:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632768 00000100:00100000:1.0:1767321247.967716:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3687 00000100:00100000:1.0:1767321247.967724:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985632768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321247.967768:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985632768:12345-0@lo:400:kworker.0 Request processed in 45us (583us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321247.967774:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3687 00000100:00080000:1.0:1767321247.967792:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321247.967984:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321247.968031:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985632768:0@lo:400:kworker.0 00000100:00100000:1.0:1767321247.968190:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632640 00000100:00100000:1.0:1767321247.968196:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632640 00000100:00100000:1.0:1767321247.968227:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3688 00000100:00080000:3.0:1767321247.968526:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321247.968557:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985632640:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321247.968609:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985632640:12345-0@lo:400:kworker.0 Request processed in 375us (625us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321247.968616:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3688 00000100:00100000:3.0:1767321247.968790:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985632640:0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.968876:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985632512:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.968913:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321247.969042:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632512 00000100:00100000:1.0:1767321247.969055:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632512 00000100:00100000:1.0:1767321247.969082:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6552 00000100:00100000:1.0:1767321247.969087:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985632512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321247.969107:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321247.969153:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985632512:12345-0@lo:38:kworker.0 Request processed in 64us (239us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321247.969189:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6552 00000100:00080000:3.0:1767321247.969579:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321247.969585:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321247.969589:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985632512:0@lo:38:kworker.0 00000100:00100000:2.0:1767321247.969907:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985632128:0@lo:400:kworker.0 00010000:00010000:1.0:1767321248.607759:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321249.627512:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321250.523502:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321250.523526:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985632896:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321250.523602:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321250.523797:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985632896 00000100:00100000:2.0:1767321250.523806:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985632896 00000100:00100000:2.0:1767321250.523882:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2912 00000100:00100000:2.0:1767321250.523893:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985632896:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321250.523923:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321250.523970:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985632896:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (367us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321250.523977:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2912 00000100:00100000:2.0:1767321250.524039:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985632896:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321250.524419:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321250.524430:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985633024:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321250.524462:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321250.524752:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633024 00000100:00100000:3.0:1767321250.524759:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633024 00000100:00100000:3.0:1767321250.524802:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2913 00000100:00100000:3.0:1767321250.524812:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985633024:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321250.524832:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321250.524880:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985633024:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (417us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321250.524887:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2913 00000100:00100000:1.0:1767321250.525141:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985633024:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321250.525329:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321250.525334:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767321250.651856:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321251.677929:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321252.699649:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321253.051131:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321253.051237:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321253.051506:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920663040 00000100:00100000:0.0:1767321253.051517:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920663040 00000100:00100000:0.0:1767321253.051590:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920662912 00000100:00100000:0.0:1767321253.051594:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920662912 00000100:00100000:0.0:1767321253.051618:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5710 00000100:00100000:0.0:1767321253.051628:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920663040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321253.051711:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920663040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (590us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321253.051719:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5710 00000100:00100000:0.0:1767321253.051740:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6553 00000100:00100000:0.0:1767321253.051744:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920662912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321253.051765:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321253.051791:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920662912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (554us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321253.051798:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6553 00000100:00080000:3.0:1767321253.052584:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321253.083540:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321253.083552:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321253.083555:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321253.083558:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321253.083563:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321253.083572:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6496 00000100:00080000:0.0:1767321253.083577:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321253.083585:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321253.083626:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb800 x1853163985633152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321253.083643:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321253.083647:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321253.083649:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321253.083652:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6496 00000100:00080000:0.0:1767321253.083655:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321253.083658:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321253.083685:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985633280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321253.083734:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321253.083738:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321253.083739:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321253.083743:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6496 00000100:00080000:0.0:1767321253.083746:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321253.083749:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321253.083763:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7480 x1853163985633536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321253.083779:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321253.083782:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321253.083784:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321253.083787:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6496 00000100:00080000:0.0:1767321253.083789:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321253.083792:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321253.083802:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b2dc2a00 x1853163985633792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321253.084006:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985633664:0@lo:400:kworker.0 00000100:00100000:2.0:1767321253.084072:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321253.084138:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321253.084144:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985633408:0@lo:400:kworker.0 00000100:00100000:2.0:1767321253.084195:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321253.084229:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985634048:0@lo:400:kworker.0 00000100:00100000:2.0:1767321253.084253:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321253.084268:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321253.084273:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985633920:0@lo:400:kworker.0 00000100:00100000:2.0:1767321253.084305:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321253.084601:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634048 00000100:00100000:1.0:1767321253.084609:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634048 00000100:00100000:1.0:1767321253.084687:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633920 00000100:00100000:1.0:1767321253.084691:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633920 00000100:00100000:1.0:1767321253.084715:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3689 00000100:00100000:1.0:1767321253.084726:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985634048:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321253.084796:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985634048:12345-0@lo:400:kworker.0 Request processed in 71us (541us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321253.084804:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3689 00000100:00100000:1.0:1767321253.084822:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3690 00000100:00100000:1.0:1767321253.084828:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985633920:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321253.084857:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985633920:12345-0@lo:400:kworker.0 Request processed in 29us (551us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321253.084862:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3690 00000100:00100000:1.0:1767321253.084894:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633664 00000100:00100000:1.0:1767321253.084897:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633664 00000100:00100000:1.0:1767321253.084920:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633408 00000100:00100000:1.0:1767321253.084923:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633408 00000100:00100000:1.0:1767321253.084936:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5711 00000100:00100000:1.0:1767321253.084941:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985633664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321253.084968:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985633664:12345-0@lo:400:kworker.0 Request processed in 27us (896us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321253.084974:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5711 00000100:00100000:1.0:1767321253.084983:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5712 00000100:00100000:1.0:1767321253.084989:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985633408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321253.085009:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985633408:12345-0@lo:400:kworker.0 Request processed in 21us (814us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321253.085014:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5712 00000100:00100000:1.0:1767321253.085047:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985633920:0@lo:400:kworker.0 00000100:00100000:1.0:1767321253.085057:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985634048:0@lo:400:kworker.0 00000100:00100000:3.0:1767321253.085327:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985633152:0@lo:38:kworker.0 00000100:00100000:3.0:1767321253.085573:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321253.085613:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985633280:0@lo:38:kworker.0 00000100:00100000:3.0:1767321253.085637:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321253.085772:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321253.085777:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321253.085779:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321253.085803:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985633408:0@lo:400:kworker.0 00000100:00100000:2.0:1767321253.085816:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985633664:0@lo:400:kworker.0 00000100:00100000:3.0:1767321253.085872:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985633536:0@lo:38:kworker.0 00000100:00100000:3.0:1767321253.085899:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321253.085925:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985633792:0@lo:38:kworker.0 00000100:00100000:3.0:1767321253.085945:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321253.085975:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633152 00000100:00100000:3.0:1767321253.085980:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633152 00000100:00100000:3.0:1767321253.085999:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633536 00000100:00100000:3.0:1767321253.086001:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633536 00000100:00100000:3.0:1767321253.086012:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633792 00000100:00100000:3.0:1767321253.086014:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633792 00000100:00100000:3.0:1767321253.086026:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6554 00000100:00100000:3.0:1767321253.086031:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985633152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321253.086048:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321253.086080:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3518e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985633152:12345-0@lo:38:kworker.0 Request processed in 47us (507us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321253.086086:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6554 00000100:00100000:3.0:1767321253.086098:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6555 00000100:00100000:3.0:1767321253.086102:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985633536:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321253.086111:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321253.086129:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985633536:12345-0@lo:38:kworker.0 Request processed in 26us (230us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321253.086138:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6555 00000100:00100000:3.0:1767321253.086147:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6556 00000100:00100000:3.0:1767321253.086151:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985633792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321253.086159:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321253.086214:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985633792:12345-0@lo:38:kworker.0 Request processed in 62us (268us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321253.086219:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6556 00000100:00080000:3.0:1767321253.086247:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321253.086252:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321253.086255:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985633152:0@lo:38:kworker.0 00000100:00080000:3.0:1767321253.086263:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321253.086266:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321253.086268:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985633536:0@lo:38:kworker.0 00000100:00080000:3.0:1767321253.086274:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321253.086277:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321253.086279:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b2dc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985633792:0@lo:38:kworker.0 00000100:00100000:1.0:1767321253.086659:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985633280 00000100:00100000:1.0:1767321253.086676:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985633280 00000100:00100000:1.0:1767321253.086706:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2129 00000100:00100000:1.0:1767321253.086711:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985633280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321253.086722:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321253.086755:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985633280:12345-0@lo:38:kworker.0 Request processed in 43us (1117us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321253.086762:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2129 00000100:00080000:3.0:1767321253.087607:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321253.087615:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321253.087619:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985633280:0@lo:38:kworker.0 00010000:00010000:0.0:1767321253.723590:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321254.747537:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321255.643602:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321255.643621:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985634176:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321255.643692:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321255.643812:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634176 00000100:00100000:1.0:1767321255.643821:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634176 00000100:00100000:1.0:1767321255.643883:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2914 00000100:00100000:1.0:1767321255.643891:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985634176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321255.643922:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321255.643982:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985634176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 91us (289us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321255.643991:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2914 00000100:00100000:1.0:1767321255.644036:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985634176:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321255.644300:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321255.644899:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321255.644913:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985634304:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321255.644965:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321255.645126:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634304 00000100:00100000:3.0:1767321255.645133:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634304 00000100:00100000:3.0:1767321255.645224:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2915 00000100:00100000:3.0:1767321255.645233:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985634304:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321255.645260:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321255.645316:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985634304:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (351us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321255.645323:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2915 00000100:00100000:0.0:1767321255.645408:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985634304:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321255.645600:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321255.774140:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321256.795463:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321257.819599:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321258.159630:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321258.159679:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321258.162285:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920663168 00000100:00100000:1.0:1767321258.162300:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920663168 00000100:00100000:1.0:1767321258.162675:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6557 00000100:00100000:1.0:1767321258.162686:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920663168:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321258.162712:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321258.163043:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920663168:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 356us (3422us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321258.163052:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6557 00000100:00100000:1.0:1767321258.163076:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920663296 00000100:00100000:1.0:1767321258.163081:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920663296 00000100:00100000:1.0:1767321258.163149:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5713 00000100:00100000:1.0:1767321258.163157:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920663296:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321258.163224:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920663296:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 68us (3544us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321258.163246:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5713 00000100:00080000:3.0:1767321258.164597:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321258.203467:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321258.203478:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321258.203482:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321258.203486:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321258.203491:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321258.203499:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6501 00000100:00080000:0.0:1767321258.203504:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321258.203510:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321258.203550:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9f80 x1853163985634432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321258.203577:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321258.203581:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321258.203583:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321258.203587:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6501 00000100:00080000:0.0:1767321258.203589:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321258.203592:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321258.203604:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdbb80 x1853163985634560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321258.203641:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321258.203644:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767321258.203645:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985634432:0@lo:38:kworker.0 00000100:00080000:0.0:1767321258.203646:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321258.203649:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6501 00000100:00080000:0.0:1767321258.203651:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321258.203654:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321258.203667:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5938e00 x1853163985634816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321258.203682:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321258.203685:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321258.203687:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321258.203689:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6501 00000100:00080000:0.0:1767321258.203691:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321258.203694:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321258.203701:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4000 x1853163985635072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321258.203709:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321258.203760:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985634560:0@lo:38:kworker.0 00000100:00100000:3.0:1767321258.203784:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321258.203810:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985634816:0@lo:38:kworker.0 00000100:00100000:3.0:1767321258.203830:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321258.203847:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985635072:0@lo:38:kworker.0 00000100:00100000:3.0:1767321258.203865:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321258.203874:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985635200:0@lo:400:kworker.0 00000100:00100000:0.0:1767321258.203902:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321258.203906:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634560 00000100:00100000:3.0:1767321258.203912:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634560 00000100:00100000:0.0:1767321258.203919:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321258.203923:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985635328:0@lo:400:kworker.0 00000100:00100000:0.0:1767321258.203941:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321258.203955:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2130 00000100:00100000:3.0:1767321258.203962:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985634560:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321258.203964:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634432 00000100:00100000:0.0:1767321258.203967:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634432 00010000:00020000:3.0:1767321258.203983:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321258.203984:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634816 00000100:00100000:0.0:1767321258.203986:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634816 00000100:00100000:0.0:1767321258.203997:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635072 00000100:00100000:0.0:1767321258.204000:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635072 00000100:00100000:0.0:1767321258.204014:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6558 00000100:00100000:0.0:1767321258.204018:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985634432:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321258.204020:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985634560:12345-0@lo:38:kworker.0 Request processed in 58us (235us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321258.204026:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2130 00010000:00020000:0.0:1767321258.204027:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321258.204050:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985634432:12345-0@lo:38:kworker.0 Request processed in 31us (341us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321258.204055:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6558 00000100:00080000:3.0:1767321258.204065:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767321258.204066:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6559 00000100:00080000:3.0:1767321258.204069:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321258.204070:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985634816:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321258.204073:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985634432:0@lo:38:kworker.0 00010000:00020000:0.0:1767321258.204078:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767321258.204081:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321258.204084:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321258.204087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985634560:0@lo:38:kworker.0 00000100:00100000:0.0:1767321258.204096:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985634816:12345-0@lo:38:kworker.0 Request processed in 25us (266us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321258.204101:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6559 00000100:00080000:3.0:1767321258.204116:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321258.204119:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321258.204122:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985634816:0@lo:38:kworker.0 00000100:00100000:0.0:1767321258.204123:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6560 00000100:00100000:0.0:1767321258.204127:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985635072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321258.204135:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321258.204185:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985635072:12345-0@lo:38:kworker.0 Request processed in 33us (296us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321258.204191:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6560 00000100:00080000:0.0:1767321258.204216:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321258.204221:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321258.204223:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985635072:0@lo:38:kworker.0 00000100:00100000:2.0:1767321258.205058:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985634688:0@lo:400:kworker.0 00000100:00100000:2.0:1767321258.205100:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321258.205254:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321258.205262:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985634944:0@lo:400:kworker.0 00000100:00100000:2.0:1767321258.205286:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321258.205317:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634688 00000100:00100000:2.0:1767321258.205322:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634688 00000100:00100000:1.0:1767321258.205472:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985634944 00000100:00100000:1.0:1767321258.205477:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985634944 00000100:00100000:2.0:1767321258.206996:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5714 00000100:00100000:2.0:1767321258.207011:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985634688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321258.207093:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985634688:12345-0@lo:400:kworker.0 Request processed in 86us (1991us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321258.207102:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5714 00000100:00100000:2.0:1767321258.207189:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985634688:0@lo:400:kworker.0 00000100:00100000:2.0:1767321258.207763:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635200 00000100:00100000:2.0:1767321258.207771:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635200 00000100:00100000:1.0:1767321258.207876:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5715 00000100:00100000:1.0:1767321258.207886:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985634944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321258.207941:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985634944:12345-0@lo:400:kworker.0 Request processed in 57us (2654us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321258.207947:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5715 00000100:00100000:1.0:1767321258.207980:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985634944:0@lo:400:kworker.0 00000100:00100000:1.0:1767321258.208000:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635328 00000100:00100000:1.0:1767321258.208003:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635328 00000100:00100000:1.0:1767321258.208030:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3691 00000100:00100000:1.0:1767321258.208036:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985635328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321258.208071:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985635328:12345-0@lo:400:kworker.0 Request processed in 36us (4129us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321258.208077:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3691 00000100:00080000:3.0:1767321258.208779:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321258.208786:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321258.208789:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321258.208906:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3692 00000100:00100000:2.0:1767321258.208915:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985635200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321258.208963:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985635200:12345-0@lo:400:kworker.0 Request processed in 50us (5059us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321258.208969:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3692 00000100:00100000:0.0:1767321258.210372:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985635328:0@lo:400:kworker.0 00000100:00100000:0.0:1767321258.210388:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985635200:0@lo:400:kworker.0 00010000:00010000:2.0:1767321258.843468:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321259.867486:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321260.763859:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321260.763889:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985635456:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321260.763957:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321260.764145:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635456 00000100:00100000:2.0:1767321260.764154:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635456 00000100:00100000:2.0:1767321260.764253:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2916 00000100:00100000:2.0:1767321260.764264:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985635456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321260.764298:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321260.764359:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985635456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (400us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321260.764367:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2916 00000100:00080000:0.0:1767321260.764511:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321260.764549:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985635456:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321260.765124:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985635584:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321260.765185:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321260.765402:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635584 00000100:00100000:3.0:1767321260.765410:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635584 00000100:00100000:3.0:1767321260.765469:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2917 00000100:00100000:3.0:1767321260.765477:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985635584:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321260.765497:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321260.765537:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985635584:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (373us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321260.765544:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2917 00000100:00080000:0.0:1767321260.767360:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321260.767430:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985635584:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321260.893679:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321261.915358:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321262.939412:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321263.279191:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321263.279245:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321263.279340:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920663552 00000100:00100000:3.0:1767321263.279341:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920663424 00000100:00100000:2.0:1767321263.279350:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920663552 00000100:00100000:3.0:1767321263.279350:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920663424 00000100:00080000:0.0:1767321263.279424:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321263.279466:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5716 00000100:00100000:3.0:1767321263.279479:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920663552:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321263.279560:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920663552:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (408us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321263.279569:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5716 00000100:00100000:3.0:1767321263.279588:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6561 00000100:00100000:3.0:1767321263.279594:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920663424:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321263.279614:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321263.279650:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920663424:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 55us (405us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321263.279657:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6561 00000100:00080000:2.0:1767321263.325275:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321263.325287:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321263.325291:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321263.325308:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321263.325313:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321263.325323:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6506 00000100:00080000:2.0:1767321263.325328:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321263.325335:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321263.325378:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75180 x1853163985635712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321263.325407:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321263.325411:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321263.325412:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321263.325415:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6506 00000100:00080000:2.0:1767321263.325418:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321263.325420:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321263.325432:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163985635840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321263.325476:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321263.325480:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321263.325481:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321263.325485:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6506 00000100:00080000:2.0:1767321263.325488:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321263.325492:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321263.325505:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985636096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321263.325513:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985635968:0@lo:400:kworker.0 00000100:00080000:2.0:1767321263.325525:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321263.325529:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321263.325531:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321263.325535:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6506 00000100:00080000:2.0:1767321263.325538:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321263.325541:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321263.325554:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74e00 x1853163985636352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321263.325588:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321263.325610:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985636224:0@lo:400:kworker.0 00000100:00100000:2.0:1767321263.325618:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985636480:0@lo:400:kworker.0 00000100:00100000:3.0:1767321263.325641:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321263.325669:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635968 00000100:00100000:1.0:1767321263.325676:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635968 00000100:00100000:2.0:1767321263.325676:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321263.325707:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321263.325714:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985636608:0@lo:400:kworker.0 00000100:00100000:2.0:1767321263.325740:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321263.325743:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5717 00000100:00100000:1.0:1767321263.325754:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985635968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321263.325773:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985636480 00000100:00100000:2.0:1767321263.325779:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985636480 00000100:00100000:1.0:1767321263.325811:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985635968:12345-0@lo:400:kworker.0 Request processed in 58us (222us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321263.325814:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985636608 00000100:00100000:2.0:1767321263.325818:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985636608 00000100:00100000:1.0:1767321263.325820:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5717 00000100:00100000:2.0:1767321263.325849:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3693 00000100:00080000:1.0:1767321263.325851:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321263.325856:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321263.325860:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985636480:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321263.325861:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321263.325880:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3694 00000100:00100000:1.0:1767321263.325889:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985636608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321263.325903:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985636480:12345-0@lo:400:kworker.0 Request processed in 46us (229us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321263.325911:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3693 00000100:00100000:3.0:1767321263.325953:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985636224 00000100:00100000:2.0:1767321263.325956:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985636608:0@lo:400:kworker.0 00000100:00100000:3.0:1767321263.325959:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985636224 00000100:00100000:1.0:1767321263.325995:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985636608:12345-0@lo:400:kworker.0 Request processed in 107us (255us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321263.325997:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5718 00000100:00100000:1.0:1767321263.326003:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3694 00000100:00100000:3.0:1767321263.326006:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985636224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321263.326020:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985635712:0@lo:38:kworker.0 00000100:00100000:1.0:1767321263.326056:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985635968:0@lo:400:kworker.0 00000100:00100000:0.0:1767321263.326066:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321263.326067:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985636224:12345-0@lo:400:kworker.0 Request processed in 62us (424us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321263.326075:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5718 00000100:00100000:3.0:1767321263.326090:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635712 00000100:00100000:3.0:1767321263.326093:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635712 00000100:00100000:0.0:1767321263.326109:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985635840:0@lo:38:kworker.0 00000100:00100000:3.0:1767321263.326116:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6562 00000100:00100000:3.0:1767321263.326125:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985635712:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767321263.326129:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:3.0:1767321263.326141:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321263.326148:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321263.326210:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985635712:12345-0@lo:38:kworker.0 Request processed in 83us (146us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321263.326214:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985636096:0@lo:38:kworker.0 00000100:00100000:3.0:1767321263.326220:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6562 00000100:00100000:0.0:1767321263.326247:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321263.326251:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985636224:0@lo:400:kworker.0 00000100:00100000:0.0:1767321263.326270:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985636352:0@lo:38:kworker.0 00000100:00100000:3.0:1767321263.326273:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985636096 00000100:00100000:3.0:1767321263.326278:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985636096 00000100:00100000:1.0:1767321263.326279:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985635840 00000100:00100000:1.0:1767321263.326283:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985635840 00000100:00100000:0.0:1767321263.326298:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321263.326302:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6563 00000100:00100000:1.0:1767321263.326303:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2131 00000100:00100000:1.0:1767321263.326309:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985635840:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321263.326309:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985636096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321263.326320:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321263.326330:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321263.326335:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321263.326339:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985635712:0@lo:38:kworker.0 00000100:00100000:0.0:1767321263.326356:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985636352 00000100:00100000:0.0:1767321263.326359:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985636352 00010000:00020000:1.0:1767321263.326375:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321263.326380:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6564 00000100:00100000:0.0:1767321263.326385:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985636352:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321263.326396:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321263.326414:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985635840:12345-0@lo:38:kworker.0 Request processed in 103us (266us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321263.326423:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2131 00000100:00100000:0.0:1767321263.326429:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985636352:12345-0@lo:38:kworker.0 Request processed in 42us (131us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321263.326436:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6564 00000100:00080000:0.0:1767321263.326467:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321263.326472:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321263.326475:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985635840:0@lo:38:kworker.0 00000100:00080000:0.0:1767321263.326482:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321263.326485:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321263.326488:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985636352:0@lo:38:kworker.0 00000100:00100000:2.0:1767321263.326848:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985636480:0@lo:400:kworker.0 00000100:00100000:3.0:1767321263.327043:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985636096:12345-0@lo:38:kworker.0 Request processed in 733us (795us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321263.327049:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6563 00000100:00080000:0.0:1767321263.328206:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321263.328214:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321263.328219:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985636096:0@lo:38:kworker.0 00010000:00010000:2.0:1767321263.965023:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321264.989011:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321265.883576:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321265.883610:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985636736:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321265.883679:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321265.886450:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985636736 00000100:00100000:2.0:1767321265.886461:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985636736 00000100:00100000:2.0:1767321265.886533:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2918 00000100:00100000:2.0:1767321265.886542:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985636736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321265.886574:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321265.886649:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985636736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 108us (2969us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321265.886656:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2918 00000100:00080000:1.0:1767321265.886740:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321265.886751:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321265.886767:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985636864:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321265.886854:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321265.887015:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985636736:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321265.887482:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985636864 00000100:00100000:2.0:1767321265.887487:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985636864 00000100:00100000:2.0:1767321265.887578:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2919 00000100:00100000:2.0:1767321265.887594:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985636864:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321265.887613:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321265.887654:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985636864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (800us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321265.887660:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2919 00000100:00100000:2.0:1767321265.887705:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985636864:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321265.900210:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767321266.016051:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321267.043126:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321267.372437:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321267.372601:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920663936 00000100:00100000:0.0:1767321267.372611:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920663936 00000100:00100000:0.0:1767321267.372692:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5719 00000100:00100000:0.0:1767321267.372706:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920663936:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321267.372790:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920663936:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (362us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321267.372798:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5719 00000100:00080000:1.0:1767321267.373103:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767321267.379244:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920663680 offset 0 length 520: 4 00000100:00100000:0.0:1767321267.379297:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321267.379380:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920663808 00000100:00100000:2.0:1767321267.379389:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920663808 00000100:00100000:2.0:1767321267.379426:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6565 00000100:00100000:2.0:1767321267.379433:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920663808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321267.379451:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321267.379552:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920663808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 117us (254us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321267.379560:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6565 00000100:00100000:1.0:1767321267.419288:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321242/real 1767321242] req@ffff8ce1a55a0a80 x1853163985631488/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321267 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321267.419323:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a55a0a80 x1853163985631488/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321267 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321267.419336:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321267.419362:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321267.419366:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985631488:0@lo:250:kworker.0 00000100:00080000:2.0:1767321267.419513:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321267.419520:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767321267.419523:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321267.419529:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6491 00000100:00080000:2.0:1767321267.419533:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321267.419540:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321267.419576:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76a00 x1853163985636992/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321267.419601:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321267.419605:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321267.419607:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321267.419610:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6511 00000100:00080000:2.0:1767321267.419613:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321267.419616:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321267.419631:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77100 x1853163985637120/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321267.419639:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321267.419642:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321267.419644:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321267.419647:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6511 00000100:00080000:2.0:1767321267.419649:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321267.419652:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321267.419664:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75c00 x1853163985637248/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321267.419704:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321267.419708:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321267.419710:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321267.419714:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6511 00000100:00080000:2.0:1767321267.419717:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321267.419720:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321267.419731:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163985637504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321267.419746:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321267.419749:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321267.419751:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321267.419754:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6511 00000100:00080000:2.0:1767321267.419757:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321267.419760:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321267.419769:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74a80 x1853163985637760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321267.419818:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985638016:0@lo:400:kworker.0 00000100:00100000:2.0:1767321267.419895:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321267.419959:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321267.419965:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985637888:0@lo:400:kworker.0 00000100:00100000:2.0:1767321267.419985:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321267.420409:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985637632:0@lo:400:kworker.0 00000100:00100000:3.0:1767321267.420447:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321267.420484:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321267.420490:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985637376:0@lo:400:kworker.0 00000100:00100000:3.0:1767321267.420510:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321267.420539:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985637632 00000100:00100000:3.0:1767321267.420546:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985637632 00000100:00100000:3.0:1767321267.420592:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985637376 00000100:00100000:3.0:1767321267.420595:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985637376 00000100:00100000:3.0:1767321267.420616:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5720 00000100:00100000:3.0:1767321267.420623:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985637632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321267.420670:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985637632:12345-0@lo:400:kworker.0 Request processed in 47us (222us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321267.420678:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5720 00000100:00100000:3.0:1767321267.420693:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5721 00000100:00100000:3.0:1767321267.420699:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985637376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321267.420726:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985637376:12345-0@lo:400:kworker.0 Request processed in 27us (215us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321267.420732:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5721 00000100:00100000:3.0:1767321267.420770:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985637376:0@lo:400:kworker.0 00000100:00100000:3.0:1767321267.420778:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985637632:0@lo:400:kworker.0 00000100:00100000:1.0:1767321267.422123:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638016 00000100:00100000:1.0:1767321267.422134:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638016 00000100:00100000:1.0:1767321267.422192:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985637888 00000100:00100000:1.0:1767321267.422195:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985637888 00000100:00100000:1.0:1767321267.422218:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3695 00000100:00100000:1.0:1767321267.422227:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985638016:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321267.422305:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985638016:12345-0@lo:400:kworker.0 Request processed in 79us (2409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321267.422313:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3695 00000100:00100000:1.0:1767321267.422327:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3696 00000100:00100000:1.0:1767321267.422333:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985637888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321267.422358:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985637888:12345-0@lo:400:kworker.0 Request processed in 25us (2372us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321267.422364:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3696 00000100:00100000:1.0:1767321267.422395:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985636992:0@lo:250:kworker.0 00000400:00000100:1.0:1767321267.422421:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985636992 offset 0 length 520: 4 00000100:00100000:1.0:1767321267.422444:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985637120:0@lo:38:kworker.0 00000100:00100000:1.0:1767321267.422468:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321267.422485:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985637248:0@lo:38:kworker.0 00000100:00100000:1.0:1767321267.422504:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321267.422520:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985637504:0@lo:38:kworker.0 00000100:00100000:1.0:1767321267.422537:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321267.422548:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985637760:0@lo:38:kworker.0 00000100:00100000:1.0:1767321267.422565:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321267.422595:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985637248 00000100:00100000:1.0:1767321267.422599:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985637248 00000100:00100000:1.0:1767321267.422617:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2132 00000100:00100000:1.0:1767321267.422621:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985637248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321267.422639:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321267.422669:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985637248:12345-0@lo:38:kworker.0 Request processed in 46us (163us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321267.422675:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2132 00000100:00080000:1.0:1767321267.422690:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321267.422694:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321267.422696:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321267.422721:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321267.422726:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321267.422729:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985637248:0@lo:38:kworker.0 00000100:00100000:1.0:1767321267.422755:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985637888:0@lo:400:kworker.0 00000100:00100000:1.0:1767321267.422766:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985638016:0@lo:400:kworker.0 00000100:00100000:1.0:1767321267.422786:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985637120 00000100:00100000:1.0:1767321267.422789:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985637120 00000100:00100000:1.0:1767321267.422805:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985637504 00000100:00100000:1.0:1767321267.422807:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985637504 00000100:00100000:1.0:1767321267.422813:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985637760 00000100:00100000:1.0:1767321267.422815:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985637760 00000100:00100000:1.0:1767321267.422823:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6566 00000100:00100000:1.0:1767321267.422827:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985637120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321267.422837:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321267.422843:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6567 00000100:00100000:0.0:1767321267.422848:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985637504:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321267.422857:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321267.422863:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985637120:12345-0@lo:38:kworker.0 Request processed in 35us (395us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321267.422869:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6566 00000100:00100000:1.0:1767321267.422880:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6568 00000100:00100000:0.0:1767321267.422881:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985637504:12345-0@lo:38:kworker.0 Request processed in 33us (344us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321267.422883:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985637760:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321267.422887:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6567 00010000:00020000:1.0:1767321267.422892:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321267.422910:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985637760:12345-0@lo:38:kworker.0 Request processed in 26us (345us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321267.422915:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6568 00000100:00080000:1.0:1767321267.422944:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321267.422949:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321267.422952:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985637120:0@lo:38:kworker.0 00000100:00080000:1.0:1767321267.422962:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321267.422966:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321267.422969:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985637504:0@lo:38:kworker.0 00000100:00080000:1.0:1767321267.422977:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321267.422980:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321267.422983:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985637760:0@lo:38:kworker.0 00010000:00010000:2.0:1767321268.061463:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321269.085617:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321270.108191:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321271.003828:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985638144:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321271.003931:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985638272:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321271.004016:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321271.004555:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638272 00000100:00100000:2.0:1767321271.004564:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638272 00000100:00100000:0.0:1767321271.004615:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321271.004657:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321271.018137:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638144 00000100:00100000:2.0:1767321271.018534:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638144 00000100:00100000:2.0:1767321271.018658:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2920 00000100:00100000:2.0:1767321271.018671:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985638272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321271.018706:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321271.018956:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985638272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 288us (14939us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321271.018965:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2920 00000100:00100000:2.0:1767321271.018984:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2921 00000100:00100000:2.0:1767321271.018991:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985638144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321271.019007:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321271.019039:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985638144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (14423us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321271.019045:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2921 00000100:00100000:2.0:1767321271.019093:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985638272:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321271.021439:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985638144:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321271.029554:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321271.131649:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321272.155671:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321272.497181:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321272.497269:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321272.497318:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664192 00000100:00100000:1.0:1767321272.497334:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664192 00000100:00100000:1.0:1767321272.497440:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664064 00000100:00100000:1.0:1767321272.497444:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664064 00000100:00100000:1.0:1767321272.497468:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5722 00000100:00100000:1.0:1767321272.497480:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920664192:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321272.497642:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6569 00000100:00100000:0.0:1767321272.497648:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920664064:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321272.497671:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321272.497747:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920664064:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 96us (477us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321272.497754:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6569 00000100:00100000:1.0:1767321272.498104:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920664192:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 626us (949us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321272.498111:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5722 00000100:00080000:3.0:1767321272.503370:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321272.540186:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321272.540196:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321272.540199:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321272.540224:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321272.540228:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321272.540236:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6515 00000100:00080000:2.0:1767321272.540241:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321272.540247:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321272.540286:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612fb80 x1853163985638400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321272.540311:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321272.540315:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321272.540317:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321272.540320:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6515 00000100:00080000:2.0:1767321272.540322:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321272.540325:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321272.540338:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612e300 x1853163985638528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321272.540387:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321272.540391:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321272.540394:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321272.540397:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6515 00000100:00080000:2.0:1767321272.540399:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321272.540403:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321272.540427:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612d500 x1853163985638784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321272.540443:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321272.540446:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321272.540448:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321272.540451:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6515 00000100:00080000:2.0:1767321272.540453:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321272.540457:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321272.540476:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c000 x1853163985639040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321272.540544:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985639168:0@lo:400:kworker.0 00000100:00100000:2.0:1767321272.540607:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321272.540670:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321272.540676:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985639296:0@lo:400:kworker.0 00000100:00100000:2.0:1767321272.540697:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321272.540938:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985638912:0@lo:400:kworker.0 00000100:00100000:0.0:1767321272.540971:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321272.541005:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321272.541011:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985638656:0@lo:400:kworker.0 00000100:00100000:0.0:1767321272.541031:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321272.544705:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639168 00000100:00100000:1.0:1767321272.544721:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639168 00000100:00100000:1.0:1767321272.544779:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639296 00000100:00100000:1.0:1767321272.544783:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639296 00000100:00100000:1.0:1767321272.544808:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3697 00000100:00100000:1.0:1767321272.544816:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985639168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321272.544915:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985639168:12345-0@lo:400:kworker.0 Request processed in 99us (4307us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321272.544922:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3697 00000100:00100000:1.0:1767321272.544937:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3698 00000100:00100000:1.0:1767321272.544944:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985639296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321272.544972:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985639296:12345-0@lo:400:kworker.0 Request processed in 29us (4273us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321272.544978:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3698 00000100:00100000:1.0:1767321272.545095:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985639296:0@lo:400:kworker.0 00000100:00100000:1.0:1767321272.545107:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985639168:0@lo:400:kworker.0 00000100:00100000:1.0:1767321272.545145:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985638400:0@lo:38:kworker.0 00000100:00100000:1.0:1767321272.545201:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321272.545232:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985638528:0@lo:38:kworker.0 00000100:00100000:1.0:1767321272.545255:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321272.545275:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985638784:0@lo:38:kworker.0 00000100:00100000:1.0:1767321272.545312:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321272.545330:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985639040:0@lo:38:kworker.0 00000100:00100000:1.0:1767321272.545346:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321272.545373:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638528 00000100:00100000:1.0:1767321272.545378:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638528 00000100:00100000:1.0:1767321272.545398:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2133 00000100:00100000:1.0:1767321272.545414:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985638528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321272.545433:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321272.546582:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638912 00000100:00100000:0.0:1767321272.546591:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638912 00000100:00100000:0.0:1767321272.546638:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638656 00000100:00100000:0.0:1767321272.546642:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638656 00000100:00100000:0.0:1767321272.546654:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638400 00000100:00100000:0.0:1767321272.546656:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638400 00000100:00100000:0.0:1767321272.546663:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985638784 00000100:00100000:0.0:1767321272.546665:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985638784 00000100:00100000:0.0:1767321272.546671:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639040 00000100:00100000:0.0:1767321272.546672:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639040 00000100:00100000:0.0:1767321272.546709:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5723 00000100:00100000:0.0:1767321272.546716:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985638912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321272.546778:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985638912:12345-0@lo:400:kworker.0 Request processed in 62us (5806us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321272.546785:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5723 00000100:00100000:0.0:1767321272.546801:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5724 00000100:00100000:0.0:1767321272.546807:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985638656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321272.546835:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985638656:12345-0@lo:400:kworker.0 Request processed in 29us (5804us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321272.546851:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5724 00000100:00100000:0.0:1767321272.546863:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6570 00000100:00100000:0.0:1767321272.546867:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985638400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321272.546880:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321272.546911:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985638400:12345-0@lo:38:kworker.0 Request processed in 42us (1709us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321272.546916:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6570 00000100:00100000:0.0:1767321272.546927:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6571 00000100:00100000:0.0:1767321272.546930:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985638784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321272.546938:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321272.546963:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985638784:12345-0@lo:38:kworker.0 Request processed in 30us (1649us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321272.546968:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6571 00000100:00100000:0.0:1767321272.546978:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6572 00000100:00100000:0.0:1767321272.546981:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985639040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321272.546989:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321272.547012:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985639040:12345-0@lo:38:kworker.0 Request processed in 28us (1664us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321272.547017:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6572 00000100:00100000:0.0:1767321272.547050:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985638656:0@lo:400:kworker.0 00000100:00100000:0.0:1767321272.547061:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985638912:0@lo:400:kworker.0 00000100:00080000:0.0:1767321272.547090:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321272.547095:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321272.547098:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985638400:0@lo:38:kworker.0 00000100:00080000:0.0:1767321272.547107:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321272.547110:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321272.547113:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985638784:0@lo:38:kworker.0 00000100:00080000:0.0:1767321272.547121:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321272.547124:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321272.547127:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985639040:0@lo:38:kworker.0 00000100:00100000:1.0:1767321272.548225:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985638528:12345-0@lo:38:kworker.0 Request processed in 2805us (2964us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321272.548235:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2133 00000100:00080000:3.0:1767321272.549082:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321272.550451:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321272.550471:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321272.550476:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985638528:0@lo:38:kworker.0 00010000:00010000:1.0:1767321273.179687:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321274.203396:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321275.227688:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321276.124251:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321276.124280:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985639424:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321276.124410:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321276.124461:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639424 00000100:00100000:3.0:1767321276.124476:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639424 00000100:00080000:1.0:1767321276.125450:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321276.128449:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2922 00000100:00100000:3.0:1767321276.128481:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985639424:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321276.128505:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321276.128538:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985639552:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321276.128654:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321276.128733:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639552 00000100:00100000:2.0:1767321276.128743:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639552 00000100:00100000:2.0:1767321276.128820:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2923 00000100:00100000:2.0:1767321276.128829:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985639552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321276.128869:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:3.0:1767321276.128870:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767321276.128880:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321276.128956:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985639552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 124us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321276.128985:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985639424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 514us (4567us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321276.129017:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985639424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321276.135603:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985639552:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321276.139766:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2923 00000100:00100000:3.0:1767321276.149106:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2922 00010000:00010000:2.0:1767321276.251502:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321277.278838:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321277.618770:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321277.620305:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321277.631337:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664448 00000100:00100000:1.0:1767321277.631353:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664448 00000100:00100000:1.0:1767321277.631482:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664320 00000100:00100000:1.0:1767321277.631487:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664320 00000100:00100000:1.0:1767321277.631523:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5725 00000100:00100000:1.0:1767321277.631536:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920664448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321277.631793:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6573 00000100:00100000:3.0:1767321277.631800:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920664320:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321277.631821:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321277.631905:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321277.632027:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920664448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 494us (13291us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321277.632044:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5725 00000100:00100000:3.0:1767321277.638953:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920664320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 7140us (18643us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321277.639475:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6573 00000100:00080000:0.0:1767321277.661453:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321277.661462:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321277.661466:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321277.661470:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321277.661474:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321277.661481:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6520 00000100:00080000:0.0:1767321277.661486:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321277.661493:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321277.661546:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ca80 x1853163985639680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321277.661578:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321277.661581:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321277.661583:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321277.661586:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6520 00000100:00080000:0.0:1767321277.661589:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321277.661592:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321277.661604:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524e300 x1853163985639808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321277.661639:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321277.661643:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321277.661645:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321277.661649:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6520 00000100:00080000:0.0:1767321277.661651:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321277.661654:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321277.661665:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524c380 x1853163985640064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321277.662049:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985639680:0@lo:38:kworker.0 00000100:00100000:1.0:1767321277.662124:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321277.662220:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985639808:0@lo:38:kworker.0 00000100:00100000:1.0:1767321277.662245:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321277.662289:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639808 00000100:00100000:1.0:1767321277.662296:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639808 00000100:00100000:1.0:1767321277.662332:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2134 00000100:00100000:1.0:1767321277.662338:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985639808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321277.662362:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321277.662404:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985639808:12345-0@lo:38:kworker.0 Request processed in 65us (157us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321277.662411:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2134 00000100:00100000:1.0:1767321277.662434:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639680 00000100:00100000:1.0:1767321277.662438:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639680 00000100:00100000:1.0:1767321277.662457:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6574 00000100:00100000:1.0:1767321277.662462:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985639680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321277.662473:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321277.662498:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985639680:12345-0@lo:38:kworker.0 Request processed in 35us (376us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321277.662504:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6574 00000100:00080000:1.0:1767321277.662535:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321277.662541:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321277.662544:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985639680:0@lo:38:kworker.0 00000100:00080000:1.0:1767321277.662553:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321277.662566:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321277.662569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985639808:0@lo:38:kworker.0 00000100:00080000:0.0:1767321277.662688:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321277.662691:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321277.662693:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321277.662697:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6520 00000100:00080000:0.0:1767321277.662700:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321277.662703:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321277.662829:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985639936:0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.662870:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321277.662893:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321277.662899:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985640192:0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.662919:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321277.662944:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985639936 00000100:00100000:2.0:1767321277.662948:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985639936 00000100:00100000:2.0:1767321277.662982:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640192 00000100:00100000:2.0:1767321277.662985:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640192 00000100:00100000:2.0:1767321277.663017:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5726 00000100:00100000:2.0:1767321277.663024:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985639936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.663068:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985639936:12345-0@lo:400:kworker.0 Request processed in 44us (196us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321277.663075:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5726 00000100:00080000:3.0:1767321277.663086:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321277.663091:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5727 00000100:00100000:2.0:1767321277.663098:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985640192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.663134:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985640192:12345-0@lo:400:kworker.0 Request processed in 35us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321277.663134:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985639936:0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.663147:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5727 00000100:00100000:3.0:1767321277.663185:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985640192:0@lo:400:kworker.0 00000100:00100000:0.0:1767321277.663326:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524df80 x1853163985640320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321277.663511:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985640448:0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.663539:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321277.663571:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321277.663577:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985640576:0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.663597:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321277.663623:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640448 00000100:00100000:2.0:1767321277.663627:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640448 00000100:00100000:2.0:1767321277.663653:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640576 00000100:00100000:2.0:1767321277.663656:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640576 00000100:00100000:2.0:1767321277.663674:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3699 00000100:00100000:2.0:1767321277.663680:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985640448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321277.663727:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985640448:12345-0@lo:400:kworker.0 Request processed in 46us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321277.663734:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3699 00000100:00100000:3.0:1767321277.663759:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3700 00000100:00100000:3.0:1767321277.663770:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985640576:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321277.663803:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985640576:12345-0@lo:400:kworker.0 Request processed in 37us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321277.665249:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985640064:0@lo:38:kworker.0 00000100:00100000:1.0:1767321277.665299:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321277.665336:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985640320:0@lo:38:kworker.0 00000100:00100000:1.0:1767321277.665358:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321277.665401:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640064 00000100:00100000:1.0:1767321277.665407:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640064 00000100:00100000:1.0:1767321277.665435:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640320 00000100:00100000:1.0:1767321277.665438:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640320 00000100:00100000:1.0:1767321277.665456:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6575 00000100:00100000:1.0:1767321277.665462:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985640064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321277.665481:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321277.665533:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985640064:12345-0@lo:38:kworker.0 Request processed in 69us (232us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321277.665541:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6575 00000100:00100000:1.0:1767321277.665557:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6576 00000100:00100000:1.0:1767321277.665561:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985640320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321277.665570:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321277.665594:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985640320:12345-0@lo:38:kworker.0 Request processed in 31us (234us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321277.665599:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6576 00000100:00080000:1.0:1767321277.665636:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321277.665642:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321277.665646:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985640064:0@lo:38:kworker.0 00000100:00080000:1.0:1767321277.665655:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321277.665658:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321277.665660:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985640320:0@lo:38:kworker.0 00000100:00100000:1.0:1767321277.665722:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985640576:0@lo:400:kworker.0 00000100:00100000:1.0:1767321277.665733:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985640448:0@lo:400:kworker.0 00000100:00100000:3.0:1767321277.678335:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3700 00010000:00010000:0.0:1767321278.300572:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321279.323409:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321280.347827:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321281.243528:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985640704:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321281.243602:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321281.243693:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321281.243700:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985640832:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321281.243720:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640704 00000100:00100000:2.0:1767321281.243728:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640704 00000100:00100000:3.0:1767321281.243730:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321281.243775:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321281.243793:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640832 00000100:00100000:2.0:1767321281.243797:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640832 00000100:00100000:3.0:1767321281.243813:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2924 00000100:00080000:0.0:1767321281.243887:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321281.243922:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985640704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321281.243955:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321281.244019:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985640704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 198us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321281.244101:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985640704:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321281.248325:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2925 00000100:00100000:2.0:1767321281.248350:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985640832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321281.248387:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321281.248478:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985640832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 140us (4746us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321281.248486:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2925 00000100:00100000:3.0:1767321281.248561:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2924 00000100:00100000:0.0:1767321281.248669:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985640832:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321281.373731:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321282.396120:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321282.734056:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321282.734113:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321282.734215:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664576 00000100:00100000:1.0:1767321282.734225:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664576 00000100:00100000:1.0:1767321282.734290:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664704 00000100:00100000:1.0:1767321282.734294:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664704 00000100:00100000:1.0:1767321282.734354:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5728 00000100:00100000:1.0:1767321282.734364:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920664704:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321282.734445:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920664704:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 83us (330us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321282.734453:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5728 00000100:00100000:1.0:1767321282.734472:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6577 00000100:00100000:1.0:1767321282.734476:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920664576:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321282.734514:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321282.734551:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920664576:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 72us (506us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321282.734556:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6577 00000100:00080000:2.0:1767321282.735089:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321282.782156:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321282.782212:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321282.782216:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321282.782220:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321282.782224:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321282.782234:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6525 00000100:00080000:3.0:1767321282.782239:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321282.782247:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321282.782287:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75180 x1853163985640960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321282.782318:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321282.782321:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321282.782323:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321282.782327:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6525 00000100:00080000:3.0:1767321282.782329:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321282.782333:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321282.782346:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74e00 x1853163985641088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321282.782386:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321282.782390:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321282.782392:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321282.782395:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6525 00000100:00080000:3.0:1767321282.782398:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321282.782402:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321282.782415:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b88e00 x1853163985641344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321282.782453:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321282.782464:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321282.782466:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321282.782469:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6525 00000100:00080000:3.0:1767321282.782472:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321282.782475:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321282.782528:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac700 x1853163985641600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321282.782609:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985641728:0@lo:400:kworker.0 00000100:00100000:3.0:1767321282.782697:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321282.782900:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321282.782907:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985641856:0@lo:400:kworker.0 00000100:00100000:3.0:1767321282.782936:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321282.783221:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985641472:0@lo:400:kworker.0 00000100:00100000:0.0:1767321282.783277:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321282.783309:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321282.783315:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985641216:0@lo:400:kworker.0 00000100:00100000:0.0:1767321282.783337:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321282.783368:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641472 00000100:00100000:0.0:1767321282.783376:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641472 00000100:00100000:0.0:1767321282.783426:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641216 00000100:00100000:0.0:1767321282.783429:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641216 00000100:00100000:0.0:1767321282.783453:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5729 00000100:00100000:0.0:1767321282.783462:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985641472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321282.783524:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985641472:12345-0@lo:400:kworker.0 Request processed in 63us (245us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321282.783532:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5729 00000100:00100000:0.0:1767321282.783676:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5730 00000100:00100000:0.0:1767321282.783686:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985641216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321282.783717:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985641216:12345-0@lo:400:kworker.0 Request processed in 34us (380us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321282.783724:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5730 00000100:00080000:0.0:1767321282.783805:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321282.783835:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985641216:0@lo:400:kworker.0 00000100:00100000:0.0:1767321282.783845:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b89f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985641472:0@lo:400:kworker.0 00000100:00100000:2.0:1767321282.784664:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641728 00000100:00100000:2.0:1767321282.784671:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641728 00000100:00100000:2.0:1767321282.784730:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641856 00000100:00100000:2.0:1767321282.784734:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641856 00000100:00100000:2.0:1767321282.784768:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3701 00000100:00100000:2.0:1767321282.784775:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985641728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321282.784827:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985641728:12345-0@lo:400:kworker.0 Request processed in 53us (2129us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321282.784835:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3701 00000100:00100000:2.0:1767321282.784848:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3702 00000100:00100000:2.0:1767321282.784854:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985641856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321282.784880:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985641856:12345-0@lo:400:kworker.0 Request processed in 26us (1943us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321282.784885:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3702 00000100:00100000:2.0:1767321282.784908:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985640960:0@lo:38:kworker.0 00000100:00100000:2.0:1767321282.784937:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321282.784962:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985641088:0@lo:38:kworker.0 00000100:00100000:2.0:1767321282.785124:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321282.785147:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985641344:0@lo:38:kworker.0 00000100:00100000:2.0:1767321282.785195:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321282.785210:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985641600:0@lo:38:kworker.0 00000100:00100000:2.0:1767321282.785236:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321282.785281:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985641856:0@lo:400:kworker.0 00000100:00100000:2.0:1767321282.785292:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985641728:0@lo:400:kworker.0 00000100:00100000:1.0:1767321282.786077:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641088 00000100:00100000:1.0:1767321282.786085:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641088 00000100:00100000:1.0:1767321282.786121:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2135 00000100:00100000:1.0:1767321282.786128:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985641088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321282.786146:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321282.786233:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985641088:12345-0@lo:38:kworker.0 Request processed in 103us (1107us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321282.786238:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2135 00000100:00080000:1.0:1767321282.786269:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321282.786274:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321282.786277:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985641088:0@lo:38:kworker.0 00000100:00100000:3.0:1767321282.786712:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985640960 00000100:00100000:3.0:1767321282.786717:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985640960 00000100:00100000:3.0:1767321282.786746:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641344 00000100:00100000:3.0:1767321282.786749:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641344 00000100:00100000:3.0:1767321282.786770:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641600 00000100:00100000:3.0:1767321282.786773:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641600 00000100:00100000:3.0:1767321282.786784:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6578 00000100:00100000:3.0:1767321282.786789:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985640960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321282.786800:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321282.786835:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985640960:12345-0@lo:38:kworker.0 Request processed in 44us (1896us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321282.786841:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6578 00000100:00100000:3.0:1767321282.786853:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6579 00000100:00100000:3.0:1767321282.786857:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985641344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321282.786867:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321282.786890:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985641344:12345-0@lo:38:kworker.0 Request processed in 31us (1694us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321282.786895:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6579 00000100:00100000:3.0:1767321282.786905:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6580 00000100:00100000:3.0:1767321282.786908:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985641600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321282.786916:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321282.786936:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985641600:12345-0@lo:38:kworker.0 Request processed in 26us (1699us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321282.786942:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6580 00000100:00080000:1.0:1767321282.789834:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321282.789847:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321282.789852:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985640960:0@lo:38:kworker.0 00000100:00080000:1.0:1767321282.789867:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321282.789870:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321282.789874:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b88e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985641344:0@lo:38:kworker.0 00000100:00080000:1.0:1767321282.789883:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321282.789886:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321282.789889:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985641600:0@lo:38:kworker.0 00010000:00010000:2.0:1767321283.419651:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321284.443401:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321285.468552:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321286.363719:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321286.363757:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985641984:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321286.363831:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321286.365503:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985641984 00000100:00100000:1.0:1767321286.365519:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985641984 00000100:00100000:1.0:1767321286.365624:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2926 00000100:00100000:1.0:1767321286.365634:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985641984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321286.365697:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321286.365769:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985641984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 136us (1936us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321286.365777:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2926 00000100:00100000:3.0:1767321286.367131:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321286.367212:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985642112:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321286.367276:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321286.367601:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321286.367653:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985641984:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321286.369066:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985642112 00000100:00100000:1.0:1767321286.369078:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985642112 00000100:00100000:1.0:1767321286.369204:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2927 00000100:00100000:1.0:1767321286.369215:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985642112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321286.369250:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321286.369333:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985642112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (2057us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321286.369341:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2927 00000100:00100000:2.0:1767321286.369448:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985642112:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321286.369899:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321286.491931:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321287.515420:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321287.851448:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321287.851703:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321287.851774:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664960 00000100:00100000:0.0:1767321287.851785:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664960 00000100:00100000:0.0:1767321287.851861:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920664832 00000100:00100000:0.0:1767321287.851865:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920664832 00000100:00100000:0.0:1767321287.851891:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5731 00000100:00100000:0.0:1767321287.851901:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920664960:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321287.851984:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920664960:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 85us (548us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321287.851996:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5731 00000100:00100000:0.0:1767321287.852020:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6581 00000100:00100000:0.0:1767321287.852025:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920664832:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321287.852046:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321287.852071:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920664832:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (368us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321287.852077:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6581 00000100:00080000:2.0:1767321287.852313:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321287.899487:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321287.899497:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321287.899501:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321287.899517:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321287.899522:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321287.899532:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6531 00000100:00080000:2.0:1767321287.899537:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321287.899543:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321287.899612:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f100 x1853163985642240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321287.899645:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321287.899649:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321287.899651:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321287.899654:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6531 00000100:00080000:2.0:1767321287.899657:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321287.899661:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321287.899673:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ea00 x1853163985642368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321287.899709:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321287.899713:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321287.899715:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321287.899719:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6531 00000100:00080000:2.0:1767321287.899721:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321287.899725:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321287.899735:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c700 x1853163985642624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321287.899759:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321287.899762:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321287.899763:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321287.899767:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6531 00000100:00080000:2.0:1767321287.899769:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321287.899772:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321287.899781:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73df80 x1853163985642880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321287.899825:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985643008:0@lo:400:kworker.0 00000100:00100000:2.0:1767321287.899901:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321287.900040:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985642240:0@lo:38:kworker.0 00000100:00100000:1.0:1767321287.900073:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321287.900131:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985642368:0@lo:38:kworker.0 00000100:00100000:1.0:1767321287.900155:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321287.900204:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985642624:0@lo:38:kworker.0 00000100:00100000:1.0:1767321287.900227:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321287.900241:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985642880:0@lo:38:kworker.0 00000100:00100000:1.0:1767321287.900259:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321287.900295:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985642368 00000100:00100000:1.0:1767321287.900302:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985642368 00000100:00100000:1.0:1767321287.900336:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2136 00000100:00100000:1.0:1767321287.900344:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985642368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321287.900364:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321287.900403:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985642368:12345-0@lo:38:kworker.0 Request processed in 58us (246us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321287.900411:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2136 00000100:00100000:1.0:1767321287.900436:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985642240 00000100:00100000:1.0:1767321287.900440:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985642240 00000100:00100000:1.0:1767321287.900457:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985642624 00000100:00100000:1.0:1767321287.900460:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985642624 00000100:00100000:1.0:1767321287.900472:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985642880 00000100:00100000:1.0:1767321287.900475:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985642880 00000100:00100000:1.0:1767321287.900489:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6582 00000100:00100000:1.0:1767321287.900493:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985642240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321287.900504:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321287.900529:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985642240:12345-0@lo:38:kworker.0 Request processed in 34us (454us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321287.900535:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6582 00000100:00100000:1.0:1767321287.900544:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6583 00000100:00100000:1.0:1767321287.900548:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985642624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321287.900568:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321287.900590:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985642624:12345-0@lo:38:kworker.0 Request processed in 40us (361us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321287.900596:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6583 00000100:00100000:1.0:1767321287.900606:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6584 00000100:00100000:1.0:1767321287.900610:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985642880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321287.900618:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321287.900636:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985642880:12345-0@lo:38:kworker.0 Request processed in 26us (376us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321287.900642:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6584 00000100:00080000:1.0:1767321287.900675:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321287.900680:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321287.900683:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985642240:0@lo:38:kworker.0 00000100:00080000:1.0:1767321287.900692:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321287.900695:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321287.900698:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985642368:0@lo:38:kworker.0 00000100:00080000:1.0:1767321287.900705:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321287.900708:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321287.900711:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985642624:0@lo:38:kworker.0 00000100:00080000:1.0:1767321287.900717:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321287.900720:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321287.900723:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985642880:0@lo:38:kworker.0 00000100:00100000:1.0:1767321287.900770:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643008 00000100:00100000:1.0:1767321287.900774:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643008 00000100:00100000:1.0:1767321287.900804:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3703 00000100:00100000:1.0:1767321287.900810:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985643008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321287.900846:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985643008:12345-0@lo:400:kworker.0 Request processed in 36us (945us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321287.900852:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3703 00000100:00100000:3.0:1767321287.900963:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985642496:0@lo:400:kworker.0 00000100:00100000:3.0:1767321287.900995:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321287.901024:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321287.901030:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985642752:0@lo:400:kworker.0 00000100:00100000:3.0:1767321287.901050:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321287.901072:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985642496 00000100:00100000:3.0:1767321287.901076:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985642496 00000100:00100000:3.0:1767321287.901107:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985642752 00000100:00100000:3.0:1767321287.901112:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985642752 00000100:00100000:3.0:1767321287.901131:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5732 00000100:00100000:3.0:1767321287.901137:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985642496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321287.901189:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985642496:12345-0@lo:400:kworker.0 Request processed in 52us (194us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321287.901198:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5732 00000100:00100000:3.0:1767321287.901210:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5733 00000100:00100000:3.0:1767321287.901216:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985642752:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321287.901240:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985642752:12345-0@lo:400:kworker.0 Request processed in 24us (189us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321287.901245:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5733 00000100:00080000:3.0:1767321287.901261:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321287.901283:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985642752:0@lo:400:kworker.0 00000100:00100000:3.0:1767321287.901292:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985642496:0@lo:400:kworker.0 00000100:00100000:2.0:1767321287.901615:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985643008:0@lo:400:kworker.0 00000100:00100000:2.0:1767321287.901629:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321287.901635:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985643136:0@lo:400:kworker.0 00000100:00100000:2.0:1767321287.901661:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321287.902479:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643136 00000100:00100000:1.0:1767321287.902486:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643136 00000100:00100000:1.0:1767321287.902544:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3704 00000100:00100000:1.0:1767321287.902553:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985643136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321287.902628:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985643136:12345-0@lo:400:kworker.0 Request processed in 76us (965us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321287.902635:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3704 00000100:00100000:2.0:1767321287.902799:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985643136:0@lo:400:kworker.0 00010000:00010000:1.0:1767321288.540275:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321289.563546:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321290.587405:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321291.483445:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321291.483469:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985643264:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321291.483609:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321291.484041:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643264 00000100:00100000:2.0:1767321291.484050:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643264 00000100:00100000:2.0:1767321291.484120:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2928 00000100:00100000:2.0:1767321291.484130:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985643264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321291.484158:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321291.484251:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985643264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 123us (641us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321291.484258:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2928 00000100:00080000:0.0:1767321291.492151:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321291.492208:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985643392:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321291.492286:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321291.492368:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643392 00000100:00100000:1.0:1767321291.492375:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643392 00000100:00100000:0.0:1767321291.492388:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985643264:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321291.492426:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2929 00000100:00100000:1.0:1767321291.492434:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985643392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321291.492458:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767321291.492472:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321291.492534:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985643392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (247us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321291.492541:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2929 00000100:00100000:0.0:1767321291.492616:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985643392:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321291.611387:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321292.643788:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767321292.972807:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920665088 offset 0 length 520: 4 00000100:00100000:2.0:1767321292.972845:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321292.972933:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321292.973048:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920665216 00000100:00100000:1.0:1767321292.973058:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920665216 00000100:00100000:1.0:1767321292.973101:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920665344 00000100:00100000:1.0:1767321292.973103:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920665344 00000100:00100000:0.0:1767321292.973192:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6585 00000100:00100000:0.0:1767321292.973208:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920665216:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321292.973229:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321292.973283:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920665216:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 83us (438us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321292.973290:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6585 00000100:00100000:1.0:1767321292.973821:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5734 00000100:00100000:1.0:1767321292.973830:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920665344:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321292.973875:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920665344:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 47us (942us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321292.973881:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5734 00000100:00080000:2.0:1767321292.974558:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321293.021837:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321293.021849:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321293.021852:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321293.021856:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321293.021859:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321293.021867:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6536 00000100:00080000:2.0:1767321293.021872:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321293.021879:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321293.021918:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f480 x1853163985643520/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321293.021936:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321293.021939:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321293.021941:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321293.021945:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6536 00000100:00080000:2.0:1767321293.021947:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321293.021950:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321293.021963:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ca80 x1853163985643648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321293.022041:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985643776:0@lo:400:kworker.0 00000100:00100000:3.0:1767321293.022108:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321293.022245:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643776 00000100:00100000:0.0:1767321293.022254:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643776 00000100:00100000:0.0:1767321293.022313:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5735 00000100:00100000:0.0:1767321293.022322:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985643776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321293.022425:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321267/real 1767321267] req@ffff8ce19db76a00 x1853163985636992/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321292 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321293.022444:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19db76a00 x1853163985636992/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321292 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321293.022457:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985643520:0@lo:38:kworker.0 00000100:00100000:1.0:1767321293.022490:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321293.022513:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985643648:0@lo:38:kworker.0 00000100:00100000:1.0:1767321293.022534:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321293.022554:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321293.022566:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321293.022570:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985636992:0@lo:250:kworker.0 00000100:00100000:1.0:1767321293.022658:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643648 00000100:00100000:1.0:1767321293.022663:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643648 00000100:00100000:1.0:1767321293.022683:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2137 00000100:00100000:1.0:1767321293.022688:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985643648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321293.022705:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321293.022735:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985643648:12345-0@lo:38:kworker.0 Request processed in 45us (199us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321293.022742:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2137 00000100:00100000:1.0:1767321293.022764:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643520 00000100:00100000:1.0:1767321293.022767:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643520 00000100:00100000:1.0:1767321293.022784:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6586 00000100:00100000:1.0:1767321293.022788:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985643520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321293.022796:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321293.022820:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985643520:12345-0@lo:38:kworker.0 Request processed in 30us (329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321293.022825:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6586 00000100:00080000:1.0:1767321293.022849:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321293.022852:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321293.022855:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985643520:0@lo:38:kworker.0 00000100:00080000:1.0:1767321293.022862:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321293.022865:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321293.022867:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985643648:0@lo:38:kworker.0 00000100:00080000:1.0:1767321293.022916:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321293.023001:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321293.023006:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321293.023008:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321293.023027:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6536 00000100:00080000:2.0:1767321293.023030:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321293.023034:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321293.023053:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e680 x1853163985643904/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321293.023218:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985644032:0@lo:400:kworker.0 00000100:00100000:1.0:1767321293.023246:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321293.023275:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985644032 00000100:00100000:1.0:1767321293.023279:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985644032 00000100:00100000:1.0:1767321293.023307:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5736 00000100:00100000:1.0:1767321293.023312:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985644032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321293.023345:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985644032:12345-0@lo:400:kworker.0 Request processed in 32us (98us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321293.023351:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5736 00000100:00100000:1.0:1767321293.023381:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985644032:0@lo:400:kworker.0 00000100:00080000:1.0:1767321293.023393:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321293.023400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985643904:0@lo:38:kworker.0 00000100:00100000:1.0:1767321293.023421:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321293.023443:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985643904 00000100:00100000:1.0:1767321293.023446:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985643904 00000100:00100000:1.0:1767321293.023475:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6587 00000100:00100000:1.0:1767321293.023479:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985643904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321293.023489:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321293.023517:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985643904:12345-0@lo:38:kworker.0 Request processed in 36us (94us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321293.023522:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6587 00000100:00080000:1.0:1767321293.023543:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321293.023547:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321293.023550:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985643904:0@lo:38:kworker.0 00000100:00080000:2.0:1767321293.027240:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321293.027251:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321293.027255:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321293.027274:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6536 00000100:00080000:2.0:1767321293.027279:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321293.027287:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321293.027326:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73fb80 x1853163985644160/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321293.027446:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985644160:0@lo:38:kworker.0 00000100:00100000:1.0:1767321293.027522:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321293.027644:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985643776:12345-0@lo:400:kworker.0 Request processed in 5321us (5533us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321293.027655:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5735 00000100:00100000:0.0:1767321293.027688:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985644160 00000100:00100000:0.0:1767321293.027694:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985644160 00000100:00100000:0.0:1767321293.027726:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6588 00000100:00100000:0.0:1767321293.027732:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985644160:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321293.027752:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321293.027855:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321293.027858:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767321293.027860:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321293.027865:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6515 00000100:00080000:2.0:1767321293.027868:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321293.027872:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321293.027882:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163985644544/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321293.027958:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985643776:0@lo:400:kworker.0 00000100:00100000:3.0:1767321293.028009:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985644288:0@lo:400:kworker.0 00000100:00100000:3.0:1767321293.028041:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321293.028065:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321293.028071:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985644416:0@lo:400:kworker.0 00000100:00100000:3.0:1767321293.028090:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321293.028126:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985644288 00000100:00100000:3.0:1767321293.028131:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985644288 00000100:00100000:3.0:1767321293.028189:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985644416 00000100:00100000:3.0:1767321293.028192:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985644416 00000100:00100000:3.0:1767321293.028228:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3705 00000100:00100000:3.0:1767321293.028235:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985644288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321293.028274:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985644288:12345-0@lo:400:kworker.0 Request processed in 40us (232us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321293.028321:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3705 00000100:00100000:3.0:1767321293.028349:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3706 00000100:00100000:3.0:1767321293.028355:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985644416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321293.028384:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985644416:12345-0@lo:400:kworker.0 Request processed in 30us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321293.028390:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3706 00000100:00100000:3.0:1767321293.028415:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985644416:0@lo:400:kworker.0 00000100:00100000:3.0:1767321293.028424:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985644288:0@lo:400:kworker.0 00000100:00080000:3.0:1767321293.028447:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321293.028451:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321293.029082:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985644544:0@lo:250:kworker.0 00000400:00000100:1.0:1767321293.029120:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985644544 offset 0 length 520: 4 00000100:00100000:0.0:1767321293.029248:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985644160:12345-0@lo:38:kworker.0 Request processed in 1513us (1723us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321293.029257:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6588 00000100:00080000:1.0:1767321293.029555:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321293.029560:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321293.029564:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985644160:0@lo:38:kworker.0 00010000:00010000:1.0:1767321293.659429:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321294.683515:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321295.707516:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321296.603485:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321296.603530:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985644672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321296.603623:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321296.603728:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321296.603741:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985644800:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321296.604037:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321296.604236:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985644672 00000100:00100000:0.0:1767321296.604247:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985644672 00000100:00100000:0.0:1767321296.604367:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985644800 00000100:00100000:0.0:1767321296.604371:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985644800 00000100:00100000:2.0:1767321296.604413:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2930 00000100:00100000:2.0:1767321296.604430:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985644672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321296.604471:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321296.604545:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985644672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 119us (922us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321296.604554:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2930 00000100:00100000:2.0:1767321296.604614:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985644672:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321296.604862:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321296.604868:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321296.604960:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2931 00000100:00100000:0.0:1767321296.604968:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985644800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321296.604987:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321296.605039:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985644800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (1002us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321296.605046:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2931 00000100:00100000:2.0:1767321296.605152:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985644800:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321296.735098:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321297.755646:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321298.093141:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321298.093250:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321298.093252:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920665600 00000100:00100000:0.0:1767321298.093266:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920665600 00000100:00100000:0.0:1767321298.093376:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920665472 00000100:00100000:0.0:1767321298.093381:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920665472 00000100:00100000:0.0:1767321298.093405:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5737 00000100:00100000:0.0:1767321298.093417:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920665600:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321298.093526:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920665600:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 111us (399us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321298.093535:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5737 00000100:00100000:0.0:1767321298.093559:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6589 00000100:00100000:0.0:1767321298.093564:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920665472:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321298.093589:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321298.093618:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920665472:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (368us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321298.093627:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6589 00000100:00080000:1.0:1767321298.093839:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321298.140410:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321298.140425:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321298.140429:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321298.140433:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321298.140439:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321298.140450:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6541 00000100:00080000:1.0:1767321298.140455:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321298.140465:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321298.140524:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7c700 x1853163985644928/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321298.140556:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321298.140560:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321298.140562:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321298.140566:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6541 00000100:00080000:1.0:1767321298.140568:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321298.140573:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321298.140588:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b7e680 x1853163985645056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321298.140629:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321298.140633:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321298.140634:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321298.140638:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6541 00000100:00080000:1.0:1767321298.140641:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321298.140644:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321298.140657:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c700 x1853163985645312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321298.140673:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321298.140676:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321298.140678:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321298.140682:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6541 00000100:00080000:1.0:1767321298.140684:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321298.140687:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321298.140698:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612f100 x1853163985645568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321298.140831:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985645440:0@lo:400:kworker.0 00000100:00100000:3.0:1767321298.140921:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321298.141001:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321298.141009:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985645184:0@lo:400:kworker.0 00000100:00100000:3.0:1767321298.141032:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321298.141119:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985645696:0@lo:400:kworker.0 00000100:00100000:2.0:1767321298.141157:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321298.141201:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321298.141207:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985645824:0@lo:400:kworker.0 00000100:00100000:2.0:1767321298.141230:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321298.141253:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645440 00000100:00100000:2.0:1767321298.141254:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985644928:0@lo:38:kworker.0 00000100:00100000:0.0:1767321298.141263:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645440 00000100:00100000:2.0:1767321298.141277:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321298.141296:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985645056:0@lo:38:kworker.0 00000100:00100000:2.0:1767321298.141319:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321298.141332:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645184 00000100:00100000:0.0:1767321298.141335:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645184 00000100:00100000:2.0:1767321298.141338:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985645312:0@lo:38:kworker.0 00000100:00100000:0.0:1767321298.141354:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985644928 00000100:00100000:0.0:1767321298.141357:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985644928 00000100:00100000:2.0:1767321298.141358:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321298.141383:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985645568:0@lo:38:kworker.0 00000100:00100000:0.0:1767321298.141399:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645312 00000100:00100000:0.0:1767321298.141402:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645312 00000100:00100000:2.0:1767321298.141403:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321298.141411:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645568 00000100:00100000:0.0:1767321298.141414:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645568 00000100:00100000:0.0:1767321298.141426:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5738 00000100:00100000:0.0:1767321298.141436:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985645440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321298.141436:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5739 00000100:00100000:2.0:1767321298.141446:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985645184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321298.141532:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645696 00000100:00100000:3.0:1767321298.141536:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645696 00000100:00100000:3.0:1767321298.141570:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645824 00000100:00100000:3.0:1767321298.141573:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645824 00000100:00100000:3.0:1767321298.141600:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3707 00000100:00100000:3.0:1767321298.141607:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985645696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321298.141623:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6590 00000100:00100000:1.0:1767321298.141629:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985644928:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321298.141644:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985645696:12345-0@lo:400:kworker.0 Request processed in 38us (487us total) trans 0 rc 0/0 00010000:00020000:1.0:1767321298.141650:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321298.141651:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3707 00000100:00100000:3.0:1767321298.141670:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3708 00000100:00100000:3.0:1767321298.141676:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985645824:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321298.141692:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985644928:12345-0@lo:38:kworker.0 Request processed in 62us (413us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321298.141698:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985645824:12345-0@lo:400:kworker.0 Request processed in 23us (468us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321298.141702:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6590 00000100:00100000:3.0:1767321298.141704:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3708 00000100:00100000:3.0:1767321298.141728:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6591 00000100:00100000:3.0:1767321298.141733:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985645312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321298.141742:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321298.141768:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985645312:12345-0@lo:38:kworker.0 Request processed in 34us (409us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321298.141774:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6591 00000100:00100000:3.0:1767321298.141787:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6592 00000100:00100000:3.0:1767321298.141791:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985645568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321298.141800:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321298.141822:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985645568:12345-0@lo:38:kworker.0 Request processed in 30us (418us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321298.141827:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6592 00000100:00100000:3.0:1767321298.141875:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985645824:0@lo:400:kworker.0 00000100:00100000:3.0:1767321298.141885:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985645696:0@lo:400:kworker.0 00000100:00080000:1.0:1767321298.141947:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321298.141952:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321298.141954:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321298.141965:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645056 00000100:00100000:1.0:1767321298.141968:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645056 00000100:00100000:1.0:1767321298.141991:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2138 00000100:00100000:1.0:1767321298.141995:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985645056:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321298.142005:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321298.142030:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985645056:12345-0@lo:38:kworker.0 Request processed in 34us (710us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321298.142036:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2138 00000100:00080000:1.0:1767321298.142071:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767321298.143862:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985645440:12345-0@lo:400:kworker.0 Request processed in 2415us (2928us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321298.143879:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5738 00000100:00100000:2.0:1767321298.143879:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ace00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985645184:12345-0@lo:400:kworker.0 Request processed in 47us (457us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321298.143893:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5739 00000100:00080000:1.0:1767321298.144199:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321298.144209:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985644928:0@lo:38:kworker.0 00000100:00080000:1.0:1767321298.144248:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321298.144253:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321298.144256:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985645056:0@lo:38:kworker.0 00000100:00080000:1.0:1767321298.144268:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321298.144271:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321298.144275:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985645312:0@lo:38:kworker.0 00000100:00080000:1.0:1767321298.144287:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321298.144291:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321298.144294:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985645568:0@lo:38:kworker.0 00000100:00100000:3.0:1767321298.145647:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985645184:0@lo:400:kworker.0 00000100:00100000:3.0:1767321298.145683:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985645440:0@lo:400:kworker.0 00010000:00010000:2.0:1767321298.779706:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321299.803514:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321300.828424:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321301.725157:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985645952:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321301.725262:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321301.725494:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321301.725503:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985646080:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321301.725527:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321301.725707:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985645952 00000100:00100000:1.0:1767321301.725714:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985645952 00000100:00100000:1.0:1767321301.725761:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646080 00000100:00100000:1.0:1767321301.725764:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646080 00000100:00100000:1.0:1767321301.725779:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2932 00000100:00100000:1.0:1767321301.725784:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985645952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321301.725810:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321301.725861:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985645952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (597us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321301.726078:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2933 00000100:00100000:2.0:1767321301.726090:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985646080:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321301.726104:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321301.726138:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985646080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (609us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321301.726144:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2933 00000100:00100000:2.0:1767321301.726234:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985646080:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321301.726242:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985645952:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321301.726298:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321301.726311:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321301.726500:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2932 00010000:00010000:2.0:1767321301.855387:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321302.875496:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321303.211432:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321303.211574:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321303.214723:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920665856 00000100:00100000:2.0:1767321303.214737:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920665856 00000100:00100000:0.0:1767321303.214795:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920665728 00000100:00100000:0.0:1767321303.214800:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920665728 00000100:00100000:2.0:1767321303.214805:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5740 00000100:00100000:2.0:1767321303.214815:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920665856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321303.214837:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6593 00000100:00100000:0.0:1767321303.214842:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920665728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321303.214863:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321303.214937:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920665856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (3518us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321303.214945:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5740 00000100:00080000:3.0:1767321303.215042:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321303.216245:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920665728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 93us (3361us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321303.216254:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6593 00000100:00080000:2.0:1767321303.259518:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321303.259528:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321303.259531:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321303.259535:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321303.259540:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321303.259547:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6546 00000100:00080000:2.0:1767321303.259551:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321303.259559:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321303.259601:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73dc00 x1853163985646208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321303.259629:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321303.259633:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321303.259635:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321303.259638:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6546 00000100:00080000:2.0:1767321303.259640:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321303.259644:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321303.259668:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73d500 x1853163985646336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321303.259710:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321303.259715:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321303.259717:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321303.259720:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6546 00000100:00080000:2.0:1767321303.259723:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321303.259726:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321303.259740:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfad80 x1853163985646592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321303.259742:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985646464:0@lo:400:kworker.0 00000100:00080000:2.0:1767321303.259756:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321303.259758:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321303.259761:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321303.259764:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6546 00000100:00080000:2.0:1767321303.259766:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321303.259769:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321303.259776:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985646848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321303.259809:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321303.259821:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985646976:0@lo:400:kworker.0 00000100:00100000:2.0:1767321303.259867:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321303.259869:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321303.259874:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985646720:0@lo:400:kworker.0 00000100:00100000:2.0:1767321303.259888:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321303.259893:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985647104:0@lo:400:kworker.0 00000100:00100000:3.0:1767321303.259895:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321303.259914:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321303.260020:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646464 00000100:00100000:0.0:1767321303.260030:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646464 00000100:00100000:0.0:1767321303.260083:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646720 00000100:00100000:0.0:1767321303.260086:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646720 00000100:00100000:0.0:1767321303.260108:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5741 00000100:00100000:0.0:1767321303.260116:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985646464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321303.260209:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985646208:0@lo:38:kworker.0 00000100:00100000:0.0:1767321303.260247:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985646464:12345-0@lo:400:kworker.0 Request processed in 131us (437us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321303.260255:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5741 00000100:00100000:0.0:1767321303.260281:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5742 00000100:00100000:0.0:1767321303.260289:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985646720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321303.261533:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646976 00000100:00100000:3.0:1767321303.261546:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646976 00000100:00100000:3.0:1767321303.261718:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985647104 00000100:00100000:3.0:1767321303.261722:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985647104 00000100:00100000:3.0:1767321303.261747:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3709 00000100:00100000:3.0:1767321303.261755:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a02450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985646976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321303.261974:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321303.262019:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985646336:0@lo:38:kworker.0 00000100:00100000:1.0:1767321303.262049:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321303.262071:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985646592:0@lo:38:kworker.0 00000100:00100000:1.0:1767321303.262091:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321303.262109:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985646848:0@lo:38:kworker.0 00000100:00100000:1.0:1767321303.262127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321303.262161:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646208 00000100:00100000:1.0:1767321303.262178:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646208 00000100:00100000:1.0:1767321303.262193:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646592 00000100:00100000:1.0:1767321303.262196:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646592 00000100:00100000:1.0:1767321303.262202:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646848 00000100:00100000:1.0:1767321303.262204:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646848 00000100:00100000:1.0:1767321303.262214:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6594 00000100:00100000:1.0:1767321303.262220:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985646208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321303.262240:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321303.262288:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985646208:12345-0@lo:38:kworker.0 Request processed in 66us (314us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321303.262295:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6594 00000100:00100000:1.0:1767321303.262309:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6595 00000100:00100000:1.0:1767321303.262313:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985646592:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321303.262322:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321303.262345:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985646592:12345-0@lo:38:kworker.0 Request processed in 31us (253us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321303.262351:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6595 00000100:00100000:1.0:1767321303.262361:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6596 00000100:00100000:1.0:1767321303.262365:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985646848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321303.262373:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321303.262395:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985646848:12345-0@lo:38:kworker.0 Request processed in 29us (267us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321303.262400:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6596 00000100:00100000:1.0:1767321303.262420:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985646336 00000100:00100000:1.0:1767321303.262423:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985646336 00000100:00100000:1.0:1767321303.262440:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2139 00000100:00100000:1.0:1767321303.262456:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985646336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321303.262468:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321303.262495:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985646336:12345-0@lo:38:kworker.0 Request processed in 38us (445us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321303.262500:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2139 00000100:00080000:1.0:1767321303.262540:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321303.262545:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321303.262549:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985646208:0@lo:38:kworker.0 00000100:00080000:1.0:1767321303.262558:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321303.262576:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321303.262579:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985646336:0@lo:38:kworker.0 00000100:00080000:1.0:1767321303.262587:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321303.262590:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321303.262592:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985646592:0@lo:38:kworker.0 00000100:00080000:1.0:1767321303.262599:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321303.262602:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321303.262605:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985646848:0@lo:38:kworker.0 00000100:00080000:1.0:1767321303.262657:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321303.262662:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321303.262664:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321303.262694:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985646464:0@lo:400:kworker.0 00000100:00100000:1.0:1767321303.262729:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3710 00000100:00100000:1.0:1767321303.262738:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985647104:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321303.262780:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985647104:12345-0@lo:400:kworker.0 Request processed in 44us (2865us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321303.262786:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3710 00000100:00100000:1.0:1767321303.262831:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985647104:0@lo:400:kworker.0 00000100:00100000:0.0:1767321303.263672:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985646720:12345-0@lo:400:kworker.0 Request processed in 3379us (3771us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321303.263682:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5742 00000100:00100000:1.0:1767321303.263899:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985646720:0@lo:400:kworker.0 00000100:00100000:3.0:1767321303.264084:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a02450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985646976:12345-0@lo:400:kworker.0 Request processed in 2327us (4214us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321303.264093:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3709 00000100:00100000:1.0:1767321303.265733:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985646976:0@lo:400:kworker.0 00010000:00010000:1.0:1767321303.899700:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321304.923435:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321305.949050:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321306.843590:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985647360:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321306.843671:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321306.843753:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321306.843759:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985647232:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321306.843784:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321306.843923:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985647360 00000100:00100000:2.0:1767321306.843932:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985647360 00000100:00100000:2.0:1767321306.843999:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985647232 00000100:00100000:2.0:1767321306.844002:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985647232 00000100:00100000:2.0:1767321306.844042:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2934 00000100:00100000:2.0:1767321306.844053:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985647360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321306.844085:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321306.844137:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985647360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (464us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321306.844145:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2934 00000100:00100000:2.0:1767321306.844184:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2935 00000100:00100000:2.0:1767321306.844192:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985647232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321306.844219:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321306.844247:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985647232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (461us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321306.844254:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2935 00000100:00100000:2.0:1767321306.844299:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985647232:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321306.844313:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985647360:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321306.844389:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321306.844395:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321306.971589:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321307.996030:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321308.329696:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321308.329747:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321308.330284:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920666112 00000100:00100000:1.0:1767321308.330295:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920666112 00000100:00100000:1.0:1767321308.330376:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920665984 00000100:00100000:1.0:1767321308.330379:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920665984 00000100:00100000:1.0:1767321308.330403:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5743 00000100:00100000:1.0:1767321308.330414:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920666112:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321308.330506:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6597 00000100:00100000:1.0:1767321308.330512:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920666112:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 98us (822us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321308.330519:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920665984:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767321308.330526:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5743 00010000:00020000:0.0:1767321308.330545:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321308.330577:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920665984:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 61us (830us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321308.330583:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6597 00000100:00080000:3.0:1767321308.330731:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321308.381032:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321308.381041:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321308.381044:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321308.381047:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321308.381051:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321308.381059:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6551 00000100:00080000:2.0:1767321308.381064:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321308.381070:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321308.381105:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb480 x1853163985647488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321308.381130:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321308.381134:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321308.381135:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321308.381139:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6551 00000100:00080000:2.0:1767321308.381141:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321308.381144:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321308.381157:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985647616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321308.381210:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321308.381214:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321308.381215:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321308.381219:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6551 00000100:00080000:2.0:1767321308.381221:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321308.381224:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321308.381237:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfa680 x1853163985647872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321308.381250:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321308.381253:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321308.381255:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321308.381257:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6551 00000100:00080000:2.0:1767321308.381259:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321308.381262:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321308.381270:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985648128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321308.381327:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985647488:0@lo:38:kworker.0 00000100:00100000:2.0:1767321308.381388:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381447:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985647616:0@lo:38:kworker.0 00000100:00100000:2.0:1767321308.381470:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381492:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985647872:0@lo:38:kworker.0 00000100:00100000:2.0:1767321308.381512:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381528:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985648128:0@lo:38:kworker.0 00000100:00100000:2.0:1767321308.381552:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381589:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985648000:0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.381622:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381639:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321308.381644:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985647744:0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.381662:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381680:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985648256:0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.381701:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381719:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321308.381724:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985648384:0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.381743:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321308.381773:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985647616 00000100:00100000:2.0:1767321308.381780:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985647616 00000100:00100000:2.0:1767321308.381812:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2140 00000100:00100000:2.0:1767321308.381819:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985647616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321308.381838:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321308.381878:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985647616:12345-0@lo:38:kworker.0 Request processed in 58us (407us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321308.381885:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2140 00000100:00100000:2.0:1767321308.381907:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648256 00000100:00100000:2.0:1767321308.381909:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648256 00000100:00100000:2.0:1767321308.381940:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648384 00000100:00100000:2.0:1767321308.381943:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648384 00000100:00100000:2.0:1767321308.381961:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3711 00000100:00100000:2.0:1767321308.381966:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985648256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.382008:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985648256:12345-0@lo:400:kworker.0 Request processed in 41us (305us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321308.382014:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3711 00000100:00100000:2.0:1767321308.382025:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3712 00000100:00100000:2.0:1767321308.382031:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985648384:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.382056:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985648384:12345-0@lo:400:kworker.0 Request processed in 26us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321308.382062:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3712 00000100:00100000:2.0:1767321308.382079:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985647488 00000100:00100000:2.0:1767321308.382082:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985647488 00000100:00100000:2.0:1767321308.382097:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985647872 00000100:00100000:2.0:1767321308.382099:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985647872 00000100:00100000:2.0:1767321308.382106:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648128 00000100:00100000:2.0:1767321308.382108:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648128 00000100:00100000:2.0:1767321308.382114:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648000 00000100:00100000:2.0:1767321308.382117:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648000 00000100:00100000:2.0:1767321308.382133:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985647744 00000100:00100000:2.0:1767321308.382135:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985647744 00000100:00100000:2.0:1767321308.382147:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5744 00000100:00100000:2.0:1767321308.382152:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985648000:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.382226:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985648000:12345-0@lo:400:kworker.0 Request processed in 73us (604us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321308.382232:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5744 00000100:00100000:2.0:1767321308.382244:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5745 00000100:00100000:2.0:1767321308.382250:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985647744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.382275:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985647744:12345-0@lo:400:kworker.0 Request processed in 26us (611us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321308.382281:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5745 00000100:00100000:2.0:1767321308.382292:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6598 00000100:00100000:2.0:1767321308.382295:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985647488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321308.382306:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321308.382332:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985647488:12345-0@lo:38:kworker.0 Request processed in 34us (944us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321308.382337:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6598 00000100:00100000:2.0:1767321308.382347:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6599 00000100:00100000:2.0:1767321308.382351:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985647872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321308.382360:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321308.382383:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985647872:12345-0@lo:38:kworker.0 Request processed in 30us (869us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321308.382388:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6599 00000100:00100000:2.0:1767321308.382399:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6600 00000100:00100000:2.0:1767321308.382402:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985648128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321308.382411:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321308.382431:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985648128:12345-0@lo:38:kworker.0 Request processed in 27us (880us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321308.382436:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6600 00000100:00100000:2.0:1767321308.382472:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985648384:0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.382484:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985648256:0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.382519:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985647744:0@lo:400:kworker.0 00000100:00100000:2.0:1767321308.382527:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985648000:0@lo:400:kworker.0 00000100:00080000:2.0:1767321308.382552:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321308.382558:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321308.382561:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985647488:0@lo:38:kworker.0 00000100:00080000:2.0:1767321308.382568:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321308.382571:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321308.382573:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985647616:0@lo:38:kworker.0 00000100:00080000:2.0:1767321308.382581:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321308.382584:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321308.382586:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985647872:0@lo:38:kworker.0 00000100:00080000:2.0:1767321308.382594:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321308.382597:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321308.382599:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985648128:0@lo:38:kworker.0 00000100:00080000:3.0:1767321308.383216:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321308.383221:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321308.383223:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767321309.019413:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321310.043404:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321311.068651:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321311.963958:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321311.963984:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985648512:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321311.964059:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321311.964278:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648512 00000100:00100000:2.0:1767321311.964288:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648512 00000100:00100000:2.0:1767321311.964355:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2936 00000100:00100000:2.0:1767321311.964365:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985648512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321311.964397:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321311.964445:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985648512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (386us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321311.964453:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2936 00000100:00080000:0.0:1767321311.964530:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321311.964571:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985648512:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321311.964914:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321311.964925:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985648640:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321311.964960:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321311.965076:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648640 00000100:00100000:2.0:1767321311.965081:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648640 00000100:00100000:2.0:1767321311.965139:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2937 00000100:00100000:2.0:1767321311.965146:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985648640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321311.965183:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321311.965226:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985648640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (265us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321311.965233:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2937 00000100:00080000:0.0:1767321311.965317:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321311.965375:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985648640:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321312.091821:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321313.115566:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321313.449657:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321313.449748:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321313.449877:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920666368 00000100:00100000:3.0:1767321313.449891:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920666368 00000100:00100000:3.0:1767321313.449958:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920666240 00000100:00100000:3.0:1767321313.450013:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920666240 00000100:00100000:3.0:1767321313.450035:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5746 00000100:00100000:3.0:1767321313.450045:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920666368:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321313.450140:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920666368:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 96us (491us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321313.450148:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5746 00000100:00100000:3.0:1767321313.450203:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6601 00000100:00100000:3.0:1767321313.450207:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920666240:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321313.450230:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321313.450256:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920666240:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (508us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321313.450263:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6601 00000100:00080000:0.0:1767321313.450334:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321313.499366:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321313.499376:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321313.499379:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321313.499382:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321313.499387:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321313.499394:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6556 00000100:00080000:3.0:1767321313.499399:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321313.499404:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321313.499442:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad880 x1853163985648768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321313.499465:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321313.499469:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321313.499471:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321313.499475:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6556 00000100:00080000:3.0:1767321313.499477:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321313.499480:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321313.499491:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ae300 x1853163985648896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321313.499539:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321313.499543:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321313.499544:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321313.499548:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6556 00000100:00080000:3.0:1767321313.499551:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321313.499554:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321313.499575:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886afb80 x1853163985649152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321313.499591:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321313.499595:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321313.499597:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321313.499600:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6556 00000100:00080000:3.0:1767321313.499602:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321313.499605:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321313.499616:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ac700 x1853163985649408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321313.499690:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985649536:0@lo:400:kworker.0 00000100:00100000:3.0:1767321313.499760:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321313.499829:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321313.499835:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985649664:0@lo:400:kworker.0 00000100:00100000:3.0:1767321313.499857:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321313.499891:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649536 00000100:00100000:3.0:1767321313.499898:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649536 00000100:00100000:3.0:1767321313.499953:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649664 00000100:00100000:3.0:1767321313.499970:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649664 00000100:00100000:3.0:1767321313.499991:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3713 00000100:00100000:3.0:1767321313.500000:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985649536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321313.500055:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985649536:12345-0@lo:400:kworker.0 Request processed in 55us (295us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321313.500062:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3713 00000100:00100000:3.0:1767321313.500079:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3714 00000100:00100000:3.0:1767321313.500095:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985649664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321313.500122:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985649664:12345-0@lo:400:kworker.0 Request processed in 38us (264us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321313.500128:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3714 00000100:00100000:3.0:1767321313.500192:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985649664:0@lo:400:kworker.0 00000100:00100000:3.0:1767321313.500204:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985649536:0@lo:400:kworker.0 00000100:00100000:2.0:1767321313.500341:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985649024:0@lo:400:kworker.0 00000100:00100000:2.0:1767321313.500374:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321313.500411:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321313.500417:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985649280:0@lo:400:kworker.0 00000100:00100000:2.0:1767321313.500438:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321313.500470:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649024 00000100:00100000:2.0:1767321313.500473:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649024 00000100:00100000:2.0:1767321313.500501:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5747 00000100:00100000:2.0:1767321313.500507:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985649024:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321313.500541:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985649024:12345-0@lo:400:kworker.0 Request processed in 34us (166us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321313.500547:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5747 00000100:00100000:2.0:1767321313.500558:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649280 00000100:00100000:2.0:1767321313.500561:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649280 00000100:00100000:2.0:1767321313.500583:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5748 00000100:00100000:2.0:1767321313.500589:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985649280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321313.500616:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985649280:12345-0@lo:400:kworker.0 Request processed in 28us (178us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321313.500622:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5748 00000100:00100000:2.0:1767321313.500662:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985649280:0@lo:400:kworker.0 00000100:00100000:2.0:1767321313.500671:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985649024:0@lo:400:kworker.0 00000100:00100000:0.0:1767321313.503346:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985648768:0@lo:38:kworker.0 00000100:00100000:0.0:1767321313.503439:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321313.503492:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985648896:0@lo:38:kworker.0 00000100:00100000:0.0:1767321313.503521:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321313.503542:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985649152:0@lo:38:kworker.0 00000100:00100000:0.0:1767321313.503686:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321313.503714:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985649408:0@lo:38:kworker.0 00000100:00100000:0.0:1767321313.503788:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321313.503839:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648768 00000100:00100000:0.0:1767321313.503849:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648768 00000100:00100000:0.0:1767321313.503884:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649152 00000100:00100000:0.0:1767321313.503887:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649152 00000100:00100000:0.0:1767321313.503897:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649408 00000100:00100000:0.0:1767321313.503899:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649408 00000100:00100000:0.0:1767321313.503913:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6602 00000100:00100000:0.0:1767321313.503920:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985648768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321313.503942:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321313.504068:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985648896 00000100:00100000:1.0:1767321313.504073:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985648896 00000100:00100000:1.0:1767321313.504107:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2141 00000100:00100000:1.0:1767321313.504112:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985648896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321313.504122:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321313.504201:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985648896:12345-0@lo:38:kworker.0 Request processed in 87us (678us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321313.504208:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2141 00000100:00080000:1.0:1767321313.504254:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321313.504259:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321313.504263:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985648896:0@lo:38:kworker.0 00000100:00080000:1.0:1767321313.504304:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321313.504310:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321313.504312:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321313.504742:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6603 00000100:00100000:2.0:1767321313.504748:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985649152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321313.504759:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321313.504810:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b89f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985649152:12345-0@lo:38:kworker.0 Request processed in 61us (1124us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321313.504815:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6603 00000100:00100000:2.0:1767321313.504833:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6604 00000100:00100000:2.0:1767321313.504837:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985649408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321313.504846:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321313.504868:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985649408:12345-0@lo:38:kworker.0 Request processed in 30us (1080us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321313.504874:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6604 00000100:00100000:0.0:1767321313.505283:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985648768:12345-0@lo:38:kworker.0 Request processed in 1360us (1842us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321313.505290:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6602 00000100:00080000:1.0:1767321313.505457:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321313.505463:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321313.505466:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985648768:0@lo:38:kworker.0 00000100:00080000:1.0:1767321313.505476:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321313.505479:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321313.505481:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985649152:0@lo:38:kworker.0 00000100:00080000:1.0:1767321313.505488:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321313.505490:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321313.505493:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985649408:0@lo:38:kworker.0 00010000:00010000:2.0:1767321314.139393:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321315.163346:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321316.190053:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321317.083499:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985649792:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321317.083590:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321317.083708:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649792 00000100:00100000:3.0:1767321317.083727:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649792 00000100:00100000:3.0:1767321317.083867:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2938 00000100:00100000:3.0:1767321317.083885:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985649792:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321317.083925:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:3.0:1767321317.083943:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321317.083945:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985649920:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321317.083984:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321317.084213:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985649792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 332us (620us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321317.084226:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2938 00000100:00100000:3.0:1767321317.084248:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985649920 00000100:00100000:3.0:1767321317.084252:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985649920 00000100:00080000:0.0:1767321317.084426:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321317.084473:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985649792:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321317.086863:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2939 00000100:00100000:2.0:1767321317.086958:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985649920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321317.087125:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321317.087334:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985649920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 396us (3348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321317.087342:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2939 00000100:00100000:0.0:1767321317.087434:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985649920:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321317.211447:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321317.548943:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:1.0:1767321317.548985:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920666496 offset 0 length 520: 4 00000100:00100000:1.0:1767321317.549004:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321317.549387:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920666752 00000100:00100000:0.0:1767321317.549401:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920666752 00000100:00100000:0.0:1767321317.549481:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920666624 00000100:00100000:0.0:1767321317.549484:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920666624 00000100:00100000:0.0:1767321317.549501:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5749 00000100:00100000:0.0:1767321317.549510:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920666752:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321317.549603:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920666752:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 95us (764us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321317.549609:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5749 00000100:00100000:0.0:1767321317.549627:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6605 00000100:00100000:0.0:1767321317.549630:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920666624:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321317.549656:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321317.549682:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920666624:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 49us (676us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321317.549690:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6605 00000100:00080000:1.0:1767321317.549869:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321318.236194:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321318.619363:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321293/real 1767321293] req@ffff8ce1af73f800 x1853163985644544/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321318 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321318.619412:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1af73f800 x1853163985644544/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321318 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767321318.619436:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321318.619528:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767321318.619532:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985644544:0@lo:250:kworker.0 00000100:00080000:1.0:1767321318.621349:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321318.621361:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767321318.621367:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321318.621377:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6541 00000100:00080000:1.0:1767321318.621383:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321318.621403:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321318.621454:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612c380 x1853163985650048/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321318.621490:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321318.621494:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321318.621496:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321318.621500:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6561 00000100:00080000:1.0:1767321318.621503:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321318.621506:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321318.621524:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612ed80 x1853163985650176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321318.621533:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321318.621536:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321318.621538:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321318.621541:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6561 00000100:00080000:1.0:1767321318.621544:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321318.621547:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321318.621561:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a612dc00 x1853163985650304/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321318.621615:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321318.621619:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321318.621621:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321318.621624:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6561 00000100:00080000:1.0:1767321318.621627:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321318.621630:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321318.621643:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e300 x1853163985650560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321318.621666:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321318.621669:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321318.621671:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321318.621673:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6561 00000100:00080000:1.0:1767321318.621675:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321318.621678:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321318.621700:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73d180 x1853163985650816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321318.621782:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985650944:0@lo:400:kworker.0 00000100:00100000:1.0:1767321318.621874:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321318.621957:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321318.621964:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985651072:0@lo:400:kworker.0 00000100:00100000:1.0:1767321318.622001:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321318.622037:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985650944 00000100:00100000:1.0:1767321318.622046:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985650944 00000100:00100000:1.0:1767321318.622101:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651072 00000100:00100000:1.0:1767321318.622104:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651072 00000100:00100000:3.0:1767321318.622132:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985650432:0@lo:400:kworker.0 00000100:00100000:1.0:1767321318.622134:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3715 00000100:00100000:1.0:1767321318.622146:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985650944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321318.622195:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321318.622236:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321318.622244:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985650944:12345-0@lo:400:kworker.0 Request processed in 98us (369us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321318.622245:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985650688:0@lo:400:kworker.0 00000100:00100000:1.0:1767321318.622259:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3715 00000100:00100000:3.0:1767321318.622278:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321318.622283:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3716 00000100:00100000:1.0:1767321318.622292:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985651072:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321318.622325:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985650432 00000100:00100000:1.0:1767321318.622330:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985651072:12345-0@lo:400:kworker.0 Request processed in 39us (338us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321318.622330:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985650432 00000100:00100000:1.0:1767321318.622339:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3716 00000100:00100000:1.0:1767321318.622390:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985651072:0@lo:400:kworker.0 00000100:00100000:3.0:1767321318.622395:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985650688 00000100:00100000:3.0:1767321318.622401:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985650688 00000100:00100000:1.0:1767321318.622406:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985650944:0@lo:400:kworker.0 00000100:00100000:3.0:1767321318.622430:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5750 00000100:00100000:3.0:1767321318.622438:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985650432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321318.622459:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5751 00000100:00100000:1.0:1767321318.622466:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985650688:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321318.622485:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985650432:12345-0@lo:400:kworker.0 Request processed in 46us (289us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321318.622493:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5750 00000100:00100000:1.0:1767321318.622498:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ac700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985650688:12345-0@lo:400:kworker.0 Request processed in 33us (220us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321318.622504:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5751 00000100:00080000:1.0:1767321318.622525:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321318.622539:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985650688:0@lo:400:kworker.0 00000100:00100000:3.0:1767321318.622552:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985650432:0@lo:400:kworker.0 00000100:00100000:0.0:1767321318.622770:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985650048:0@lo:250:kworker.0 00000400:00000100:0.0:1767321318.622801:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985650048 offset 0 length 520: 4 00000100:00100000:0.0:1767321318.622823:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985650176:0@lo:38:kworker.0 00000100:00100000:0.0:1767321318.622851:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321318.622879:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985650304:0@lo:38:kworker.0 00000100:00100000:0.0:1767321318.622906:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321318.622932:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985650560:0@lo:38:kworker.0 00000100:00100000:0.0:1767321318.622954:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321318.622973:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985650304 00000100:00100000:0.0:1767321318.622977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985650816:0@lo:38:kworker.0 00000100:00100000:3.0:1767321318.622981:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985650304 00000100:00100000:0.0:1767321318.623016:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321318.623030:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2142 00000100:00100000:3.0:1767321318.623037:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985650304:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321318.623052:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985650176 00000100:00100000:0.0:1767321318.623057:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985650176 00010000:00020000:3.0:1767321318.623062:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321318.623076:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985650560 00000100:00100000:0.0:1767321318.623080:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985650560 00000100:00100000:0.0:1767321318.623095:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985650816 00000100:00100000:0.0:1767321318.623098:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985650816 00000100:00100000:3.0:1767321318.623110:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985650304:12345-0@lo:38:kworker.0 Request processed in 71us (201us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321318.623117:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6606 00000100:00100000:3.0:1767321318.623122:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2142 00000100:00100000:0.0:1767321318.623127:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985650176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321318.623140:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321318.623154:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6607 00000100:00100000:0.0:1767321318.623196:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985650176:12345-0@lo:38:kworker.0 Request processed in 70us (342us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321318.623202:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6606 00000100:00100000:0.0:1767321318.623217:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6608 00000100:00100000:0.0:1767321318.623221:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985650816:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321318.623230:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321318.623255:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985650816:12345-0@lo:38:kworker.0 Request processed in 32us (239us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321318.623261:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6608 00000100:00080000:0.0:1767321318.623305:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321318.623311:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321318.623316:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985650176:0@lo:38:kworker.0 00000100:00080000:0.0:1767321318.623324:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321318.623327:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321318.623329:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985650304:0@lo:38:kworker.0 00000100:00080000:0.0:1767321318.623336:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321318.623340:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321318.623343:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985650816:0@lo:38:kworker.0 00000100:00100000:3.0:1767321318.623889:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985650560:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321318.623909:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321318.623967:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985650560:12345-0@lo:38:kworker.0 Request processed in 804us (1012us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321318.623976:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6607 00000100:00080000:0.0:1767321318.624019:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321318.624024:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321318.624028:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985650560:0@lo:38:kworker.0 00010000:00010000:1.0:1767321319.264764:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321320.283684:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321321.309112:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321322.203519:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985651200:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321322.203621:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321322.203672:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985651328:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321322.203691:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321322.203772:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651200 00000100:00100000:2.0:1767321322.203773:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651328 00000100:00100000:1.0:1767321322.203784:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651200 00000100:00100000:2.0:1767321322.203784:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651328 00000100:00100000:2.0:1767321322.203867:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2940 00000100:00100000:2.0:1767321322.203879:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985651328:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321322.203916:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321322.208017:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2941 00000100:00100000:1.0:1767321322.208040:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985651200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321322.208091:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767321322.211767:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321322.211783:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321322.213946:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985651328:12345-0@lo:13:osp-pre-0-1.0 Request processed in 10044us (10228us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321322.213966:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2940 00000100:00100000:1.0:1767321322.214718:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886afb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985651200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 6683us (11092us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321322.214730:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2941 00000100:00100000:3.0:1767321322.217725:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985651200:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321322.217759:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985651328:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321322.332438:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321322.666751:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321322.667129:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920667008 00000100:00100000:3.0:1767321322.667144:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920667008 00000100:00100000:1.0:1767321322.667256:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:0.0:1767321322.667467:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321322.677338:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920666880 00000100:00100000:3.0:1767321322.677354:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920666880 00000100:00100000:3.0:1767321322.677511:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5752 00000100:00100000:3.0:1767321322.677528:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920667008:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321322.677641:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920667008:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 108us (10895us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321322.677650:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5752 00000100:00100000:3.0:1767321322.677670:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6609 00000100:00100000:3.0:1767321322.677674:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920666880:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321322.677695:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321322.677721:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920666880:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (10469us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321322.677728:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6609 00010000:00010000:2.0:1767321323.359311:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321323.739521:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321323.739535:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321323.739539:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321323.739544:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321323.739549:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321323.739561:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6566 00000100:00080000:2.0:1767321323.739567:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321323.739576:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321323.739636:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2d80 x1853163985651456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321323.739668:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321323.739672:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321323.739674:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321323.739678:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6566 00000100:00080000:2.0:1767321323.739681:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321323.739684:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321323.739699:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0e00 x1853163985651584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321323.739787:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985651712:0@lo:400:kworker.0 00000100:00100000:3.0:1767321323.739884:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321323.739968:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985651456:0@lo:38:kworker.0 00000100:00100000:0.0:1767321323.740003:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321323.740029:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767321323.740033:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985651584:0@lo:38:kworker.0 00000100:00080000:2.0:1767321323.740034:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321323.740036:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321323.740040:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6566 00000100:00080000:2.0:1767321323.740043:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321323.740047:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321323.740063:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2a00 x1853163985651840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321323.740093:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321323.740097:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321323.740099:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321323.740102:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6566 00000100:00080000:2.0:1767321323.740104:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321323.740108:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321323.740119:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0380 x1853163985652096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321323.740216:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321323.740223:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985652352:0@lo:400:kworker.0 00000100:00100000:1.0:1767321323.740279:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651712 00000100:00100000:1.0:1767321323.740290:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651712 00000100:00100000:1.0:1767321323.740371:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651456 00000100:00100000:1.0:1767321323.740376:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651456 00000100:00100000:0.0:1767321323.740410:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321323.740522:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5753 00000100:00100000:1.0:1767321323.740551:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6610 00000100:00100000:2.0:1767321323.740552:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985651712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321323.740576:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985651456:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321323.740609:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321323.740671:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321323.740728:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985651456:12345-0@lo:38:kworker.0 Request processed in 150us (718us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321323.740746:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6610 00000100:00100000:3.0:1767321323.740749:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651584 00000100:00100000:3.0:1767321323.740756:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651584 00000100:00080000:1.0:1767321323.740787:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321323.740795:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2143 00000100:00100000:3.0:1767321323.740806:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190a12050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985651584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321323.740824:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321323.740864:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190a12050 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985651584:12345-0@lo:38:kworker.0 Request processed in 60us (516us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321323.740870:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2143 00000100:00100000:1.0:1767321323.741038:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985652224:0@lo:400:kworker.0 00000100:00100000:1.0:1767321323.741084:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321323.741121:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321323.741130:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985651968:0@lo:400:kworker.0 00000100:00100000:1.0:1767321323.741160:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321323.741240:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651968 00000100:00100000:1.0:1767321323.741245:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652352 00000100:00100000:3.0:1767321323.741249:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651968 00000100:00100000:1.0:1767321323.741251:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652352 00000100:00100000:1.0:1767321323.741290:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652224 00000100:00100000:1.0:1767321323.741294:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652224 00000100:00100000:3.0:1767321323.741304:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5754 00000100:00100000:3.0:1767321323.741314:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985651968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321323.741324:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3717 00000100:00100000:1.0:1767321323.741334:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985652352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321323.741375:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985651968:12345-0@lo:400:kworker.0 Request processed in 63us (215us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321323.741384:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5754 00000100:00100000:1.0:1767321323.741391:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886aed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985652352:12345-0@lo:400:kworker.0 Request processed in 57us (721us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321323.741399:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3717 00000100:00100000:3.0:1767321323.741414:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3718 00000100:00100000:3.0:1767321323.741423:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985652224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321323.741452:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985652224:12345-0@lo:400:kworker.0 Request processed in 31us (368us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321323.741555:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985651968:0@lo:400:kworker.0 00000100:00100000:1.0:1767321323.741566:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985652224:0@lo:400:kworker.0 00000100:00100000:1.0:1767321323.741600:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985652352:0@lo:400:kworker.0 00000100:00080000:1.0:1767321323.741632:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321323.741792:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3718 00000100:00080000:0.0:1767321323.743283:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321323.743292:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321323.743296:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985651584:0@lo:38:kworker.0 00000100:00100000:0.0:1767321323.743319:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985651840:0@lo:38:kworker.0 00000100:00100000:0.0:1767321323.743368:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321323.743398:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985652096:0@lo:38:kworker.0 00000100:00100000:0.0:1767321323.743415:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321323.743444:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321323.743447:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321323.743449:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985651456:0@lo:38:kworker.0 00000100:00100000:1.0:1767321323.743520:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985651840 00000100:00100000:1.0:1767321323.743535:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985651840 00000100:00100000:1.0:1767321323.743591:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652096 00000100:00100000:1.0:1767321323.743594:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652096 00000100:00100000:1.0:1767321323.743617:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6611 00000100:00100000:1.0:1767321323.743626:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985651840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321323.743653:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321323.743732:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985651840:12345-0@lo:38:kworker.0 Request processed in 104us (361us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321323.743742:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6611 00000100:00100000:3.0:1767321323.743760:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6612 00000100:00100000:3.0:1767321323.743770:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985652096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321323.743791:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321323.743839:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985652096:12345-0@lo:38:kworker.0 Request processed in 69us (422us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321323.744037:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ae300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985651712:12345-0@lo:400:kworker.0 Request processed in 3499us (4150us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321323.744048:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5753 00000100:00100000:1.0:1767321323.744278:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985651712:0@lo:400:kworker.0 00000100:00080000:0.0:1767321323.744292:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321323.744299:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321323.744304:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985651840:0@lo:38:kworker.0 00000100:00080000:0.0:1767321323.744318:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321323.744322:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321323.744326:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985652096:0@lo:38:kworker.0 00000100:00100000:3.0:1767321323.749716:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6612 00010000:00010000:1.0:1767321324.379418:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321325.406154:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321326.427549:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321327.323364:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985652480:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321327.323437:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321327.323469:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652480 00000100:00100000:0.0:1767321327.323479:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652480 00000100:00100000:0.0:1767321327.323555:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2942 00000100:00100000:0.0:1767321327.323565:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985652480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321327.323594:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321327.323644:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985652480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (206us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321327.323653:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2942 00000100:00100000:0.0:1767321327.323698:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985652480:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321327.323767:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321327.323895:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985652608:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321327.323924:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321327.323986:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652608 00000100:00100000:2.0:1767321327.323991:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652608 00000100:00100000:2.0:1767321327.324036:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2943 00000100:00100000:2.0:1767321327.324044:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985652608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321327.324063:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767321327.324102:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321327.324202:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985652608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 159us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321327.324210:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2943 00000100:00100000:1.0:1767321327.324280:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985652608:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767321327.451503:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321327.792263:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321327.792311:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321327.792590:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920667136 00000100:00100000:1.0:1767321327.792601:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920667136 00000100:00100000:1.0:1767321327.792648:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920667264 00000100:00100000:1.0:1767321327.792651:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920667264 00000100:00100000:1.0:1767321327.792712:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5755 00000100:00100000:1.0:1767321327.792722:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920667264:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321327.792800:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920667264:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (488us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321327.792807:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5755 00000100:00100000:1.0:1767321327.792835:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6613 00000100:00100000:1.0:1767321327.792840:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920667136:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321327.792860:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321327.792886:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920667136:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (637us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321327.792893:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6613 00000100:00080000:2.0:1767321327.793263:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321328.476280:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321328.859379:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321328.859388:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321328.859391:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321328.859395:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321328.859400:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321328.859409:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6571 00000100:00080000:3.0:1767321328.859413:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321328.859419:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321328.859458:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886ad180 x1853163985652736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321328.859486:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321328.859489:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321328.859491:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321328.859495:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6571 00000100:00080000:3.0:1767321328.859497:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321328.859500:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321328.859510:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1886af480 x1853163985652864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321328.859550:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321328.859554:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321328.859556:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321328.859559:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6571 00000100:00080000:3.0:1767321328.859561:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321328.859564:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321328.859575:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5500 x1853163985653120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321328.859597:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321328.859600:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321328.859602:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321328.859605:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6571 00000100:00080000:3.0:1767321328.859607:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321328.859609:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321328.859618:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a680 x1853163985653376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321328.859657:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985653248:0@lo:400:kworker.0 00000100:00100000:3.0:1767321328.859712:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321328.859769:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321328.859774:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985653504:0@lo:400:kworker.0 00000100:00100000:3.0:1767321328.859798:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321328.859857:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985653248 00000100:00100000:3.0:1767321328.859864:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985653248 00000100:00100000:1.0:1767321328.859909:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985652992:0@lo:400:kworker.0 00000100:00100000:3.0:1767321328.859913:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5756 00000100:00100000:3.0:1767321328.859921:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985653248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321328.859938:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321328.859957:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321328.859962:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985653632:0@lo:400:kworker.0 00000100:00100000:3.0:1767321328.859977:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985653248:12345-0@lo:400:kworker.0 Request processed in 57us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321328.859980:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321328.859984:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5756 00000100:00100000:3.0:1767321328.859998:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652992 00000100:00100000:3.0:1767321328.860001:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652992 00000100:00100000:1.0:1767321328.860009:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985653504 00000100:00100000:1.0:1767321328.860012:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985653504 00000100:00100000:3.0:1767321328.860021:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5757 00000100:00100000:3.0:1767321328.860026:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985652992:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321328.860033:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985653632 00000100:00100000:1.0:1767321328.860035:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985653632 00000100:00100000:1.0:1767321328.860051:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3719 00000100:00100000:3.0:1767321328.860054:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985652992:12345-0@lo:400:kworker.0 Request processed in 28us (116us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321328.860057:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985653504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321328.860060:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5757 00000100:00100000:3.0:1767321328.860076:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3720 00000100:00100000:3.0:1767321328.860082:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985653632:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321328.860083:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985653504:12345-0@lo:400:kworker.0 Request processed in 27us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321328.860089:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3719 00000100:00100000:3.0:1767321328.860106:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985653632:12345-0@lo:400:kworker.0 Request processed in 23us (124us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321328.860112:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3720 00000100:00100000:1.0:1767321328.860129:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985653632:0@lo:400:kworker.0 00000100:00100000:3.0:1767321328.860132:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985653504:0@lo:400:kworker.0 00000100:00100000:1.0:1767321328.860138:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985652992:0@lo:400:kworker.0 00000100:00100000:3.0:1767321328.860140:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985653248:0@lo:400:kworker.0 00000100:00080000:2.0:1767321328.860215:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321328.860269:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985652736:0@lo:38:kworker.0 00000100:00100000:0.0:1767321328.860296:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321328.860322:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985652864:0@lo:38:kworker.0 00000100:00100000:0.0:1767321328.860382:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321328.860409:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985653120:0@lo:38:kworker.0 00000100:00100000:1.0:1767321328.860422:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652864 00000100:00100000:1.0:1767321328.860426:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652864 00000100:00100000:0.0:1767321328.860429:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321328.860447:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2144 00000100:00100000:0.0:1767321328.860452:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985653376:0@lo:38:kworker.0 00000100:00100000:1.0:1767321328.860452:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985652864:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321328.860470:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767321328.860470:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321328.860495:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985652736 00000100:00100000:0.0:1767321328.860498:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985652736 00000100:00100000:1.0:1767321328.860500:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985652864:12345-0@lo:38:kworker.0 Request processed in 48us (118us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321328.860506:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2144 00000100:00100000:0.0:1767321328.860513:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985653120 00000100:00100000:0.0:1767321328.860516:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985653120 00000100:00100000:0.0:1767321328.860532:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985653376 00000100:00100000:0.0:1767321328.860535:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985653376 00000100:00080000:1.0:1767321328.860541:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321328.860546:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321328.860548:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6614 00000100:00100000:1.0:1767321328.860549:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985652864:0@lo:38:kworker.0 00000100:00100000:0.0:1767321328.860551:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985652736:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321328.860560:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321328.860567:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6615 00000100:00100000:1.0:1767321328.860571:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985653120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321328.860580:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321328.860585:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985652736:12345-0@lo:38:kworker.0 Request processed in 33us (288us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321328.860591:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6614 00000100:00100000:1.0:1767321328.860599:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985653120:12345-0@lo:38:kworker.0 Request processed in 27us (170us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321328.860603:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6616 00000100:00100000:0.0:1767321328.860606:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985653376:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321328.860606:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6615 00010000:00020000:0.0:1767321328.860615:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321328.860635:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985653376:12345-0@lo:38:kworker.0 Request processed in 27us (163us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321328.860640:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6616 00000100:00080000:1.0:1767321328.860643:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321328.860647:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321328.860650:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1886ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985652736:0@lo:38:kworker.0 00000100:00080000:1.0:1767321328.860657:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321328.860660:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321328.860662:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985653120:0@lo:38:kworker.0 00000100:00080000:1.0:1767321328.860669:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321328.860672:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321328.860675:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985653376:0@lo:38:kworker.0 00010000:00010000:2.0:1767321329.499336:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321330.523438:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321331.556836:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321332.443551:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985653760:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321332.443645:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321332.443883:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321332.443895:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985653888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321332.443928:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321332.443966:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985653760 00000100:00100000:0.0:1767321332.443979:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985653760 00000100:00100000:1.0:1767321332.444001:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985653888 00000100:00100000:1.0:1767321332.444006:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985653888 00000100:00100000:0.0:1767321332.444047:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2944 00000100:00080000:2.0:1767321332.444053:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321332.444056:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985653760:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321332.444057:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:0.0:1767321332.444087:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321332.444140:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985653760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (495us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321332.444147:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2944 00000100:00100000:0.0:1767321332.444183:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2945 00000100:00100000:0.0:1767321332.444190:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985653888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321332.444204:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321332.444233:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985653888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (304us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321332.444239:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2945 00000100:00100000:2.0:1767321332.444311:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985653888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321332.444342:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985653760:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321332.573907:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321332.908533:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321332.908588:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321332.908690:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920667520 00000100:00100000:2.0:1767321332.908705:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920667520 00000100:00100000:2.0:1767321332.908818:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920667392 00000100:00100000:2.0:1767321332.908822:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920667392 00000100:00100000:2.0:1767321332.908846:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5758 00000100:00100000:2.0:1767321332.908860:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920667520:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321332.908983:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920667520:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 126us (467us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321332.908992:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5758 00000100:00100000:2.0:1767321332.909016:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6617 00000100:00100000:2.0:1767321332.909021:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920667392:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321332.909048:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321332.909081:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920667392:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 57us (492us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321332.909088:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6617 00000100:00080000:0.0:1767321332.909395:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767321333.595444:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321333.979398:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321333.979413:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321333.979418:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321333.979423:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321333.979428:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321333.979451:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6577 00000100:00080000:2.0:1767321333.979457:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321333.979465:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321333.979522:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3100 x1853163985654016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321333.979557:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321333.979561:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321333.979563:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321333.979567:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6577 00000100:00080000:2.0:1767321333.979570:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321333.979574:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321333.979596:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d2d80 x1853163985654144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321333.979645:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321333.979649:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321333.979651:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321333.979655:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6577 00000100:00080000:2.0:1767321333.979657:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321333.979660:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321333.979673:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1880 x1853163985654400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321333.979728:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321333.979733:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321333.979737:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767321333.979741:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985654272:0@lo:400:kworker.0 00000100:00080000:2.0:1767321333.979743:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6577 00000100:00080000:2.0:1767321333.979749:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321333.979758:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321333.979774:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0a80 x1853163985654656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321333.979830:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985654528:0@lo:400:kworker.0 00000100:00100000:1.0:1767321333.979899:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985654016:0@lo:38:kworker.0 00000100:00100000:0.0:1767321333.981233:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985654912:0@lo:400:kworker.0 00000100:00100000:2.0:1767321333.981521:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321333.981612:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321333.981621:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985654784:0@lo:400:kworker.0 00000100:00100000:2.0:1767321333.981651:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321333.981732:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654528 00000100:00100000:2.0:1767321333.981744:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654528 00000100:00100000:2.0:1767321333.981828:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5759 00000100:00100000:2.0:1767321333.981844:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985654528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321333.981914:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985654528:12345-0@lo:400:kworker.0 Request processed in 75us (394us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321333.981921:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5759 00000100:00100000:2.0:1767321333.981973:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985654528:0@lo:400:kworker.0 00000100:00100000:2.0:1767321333.982006:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654784 00000100:00100000:2.0:1767321333.982009:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654784 00000100:00100000:2.0:1767321333.982040:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3721 00000100:00100000:2.0:1767321333.982046:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985654784:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321333.982080:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985654784:12345-0@lo:400:kworker.0 Request processed in 35us (428us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321333.982087:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3721 00000100:00100000:2.0:1767321333.982119:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985654784:0@lo:400:kworker.0 00000100:00080000:2.0:1767321333.982141:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321333.982423:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321333.982452:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985654144:0@lo:38:kworker.0 00000100:00100000:1.0:1767321333.982481:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321333.982508:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985654400:0@lo:38:kworker.0 00000100:00100000:1.0:1767321333.982530:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321333.982544:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985654656:0@lo:38:kworker.0 00000100:00100000:1.0:1767321333.982564:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321333.982599:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654016 00000100:00100000:1.0:1767321333.982603:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654016 00000100:00100000:1.0:1767321333.982623:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654400 00000100:00100000:1.0:1767321333.982626:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654400 00000100:00100000:1.0:1767321333.982638:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654656 00000100:00100000:1.0:1767321333.982641:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654656 00000100:00100000:1.0:1767321333.982656:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6618 00000100:00100000:1.0:1767321333.982662:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985654016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321333.982688:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321333.984763:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886ad180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985654016:12345-0@lo:38:kworker.0 Request processed in 2084us (2324us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321333.984779:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6618 00000100:00100000:1.0:1767321333.984825:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6619 00000100:00100000:1.0:1767321333.984832:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985654400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321333.984857:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321333.984901:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886adc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985654400:12345-0@lo:38:kworker.0 Request processed in 67us (2369us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321333.984906:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6619 00000100:00100000:1.0:1767321333.984918:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6620 00000100:00100000:1.0:1767321333.984923:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985654656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321333.984933:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321333.984956:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985654656:12345-0@lo:38:kworker.0 Request processed in 32us (2392us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321333.984962:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6620 00000100:00080000:1.0:1767321333.985027:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321333.985034:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321333.985038:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985654016:0@lo:38:kworker.0 00000100:00080000:1.0:1767321333.985050:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321333.985054:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321333.985057:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985654400:0@lo:38:kworker.0 00000100:00080000:1.0:1767321333.985067:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321333.985071:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321333.985074:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985654656:0@lo:38:kworker.0 00000100:00100000:3.0:1767321333.985235:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321333.985274:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654144 00000100:00100000:2.0:1767321333.985285:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654144 00000100:00100000:2.0:1767321333.985330:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2145 00000100:00100000:2.0:1767321333.985336:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985654144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321333.985350:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321333.985359:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654272 00000100:00100000:3.0:1767321333.985370:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654272 00000100:00100000:2.0:1767321333.985400:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1886af480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985654144:12345-0@lo:38:kworker.0 Request processed in 62us (2917us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321333.985408:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2145 00000100:00080000:2.0:1767321333.985458:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321333.985467:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5760 00000100:00100000:3.0:1767321333.985483:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985654272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321333.987624:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985654272:12345-0@lo:400:kworker.0 Request processed in 2137us (2388us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321333.987638:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5760 00000100:00100000:2.0:1767321333.987847:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985654272:0@lo:400:kworker.0 00000100:00080000:1.0:1767321333.988336:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321333.988344:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321333.988349:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985654144:0@lo:38:kworker.0 00000100:00100000:0.0:1767321333.988686:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321333.988809:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985654912 00000100:00100000:2.0:1767321333.988818:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985654912 00000100:00100000:2.0:1767321333.988888:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3722 00000100:00100000:2.0:1767321333.988900:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985654912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321333.988980:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985654912:12345-0@lo:400:kworker.0 Request processed in 83us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321333.988988:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3722 00000100:00100000:0.0:1767321333.989067:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985654912:0@lo:400:kworker.0 00010000:00010000:1.0:1767321334.619963:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321335.646572:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321336.669512:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321337.563537:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985655168:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321337.563616:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321337.563628:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985655040:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321337.563666:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321337.563703:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655168 00000100:00100000:0.0:1767321337.563711:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655168 00000100:00100000:3.0:1767321337.563746:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655040 00000100:00100000:3.0:1767321337.563751:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655040 00000100:00100000:0.0:1767321337.563770:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2946 00000100:00100000:3.0:1767321337.563775:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2947 00000100:00100000:0.0:1767321337.563777:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985655168:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321337.563781:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985655040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321337.563804:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:3.0:1767321337.563804:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321337.563850:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985655168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (234us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321337.563851:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985655040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (185us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321337.563859:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2947 00000100:00080000:2.0:1767321337.563895:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321337.563899:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321337.563905:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985655040:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321337.563919:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985655168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321337.568767:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2946 00010000:00010000:1.0:1767321337.692579:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321338.026978:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321338.027032:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321338.027143:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920667776 00000100:00100000:3.0:1767321338.027153:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920667776 00000100:00100000:2.0:1767321338.027204:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920667648 00000100:00100000:2.0:1767321338.027209:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920667648 00000100:00080000:0.0:1767321338.027281:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321338.027402:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5761 00000100:00100000:2.0:1767321338.027414:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920667776:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321338.027490:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920667776:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (523us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321338.027499:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5761 00000100:00100000:2.0:1767321338.027519:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6621 00000100:00100000:2.0:1767321338.027523:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920667648:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321338.027544:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321338.027567:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920667648:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (536us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321338.027573:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6621 00010000:00010000:2.0:1767321338.716410:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321339.099627:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321339.099638:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321339.099641:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321339.099645:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321339.099649:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321339.099657:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6582 00000100:00080000:3.0:1767321339.099662:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321339.099670:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321339.099715:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578380 x1853163985655296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321339.099740:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321339.099743:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321339.099745:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321339.099748:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6582 00000100:00080000:3.0:1767321339.099751:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321339.099754:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321339.099769:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a680 x1853163985655424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321339.099812:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321339.099816:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321339.099818:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321339.099821:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6582 00000100:00080000:3.0:1767321339.099824:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321339.099827:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321339.099838:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1500 x1853163985655680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321339.099864:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321339.099867:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321339.099869:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321339.099872:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6582 00000100:00080000:3.0:1767321339.099875:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321339.099878:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321339.099888:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3fff480 x1853163985655936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321339.099934:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985655808:0@lo:400:kworker.0 00000100:00100000:3.0:1767321339.099996:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321339.100042:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321339.100046:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985656064:0@lo:400:kworker.0 00000100:00100000:3.0:1767321339.100063:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321339.100089:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985656064 00000100:00100000:3.0:1767321339.100096:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985656064 00000100:00100000:3.0:1767321339.100146:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3723 00000100:00100000:3.0:1767321339.100154:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985656064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321339.100236:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985656064:12345-0@lo:400:kworker.0 Request processed in 83us (170us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321339.100242:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3723 00000100:00080000:3.0:1767321339.100260:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321339.100373:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985655552:0@lo:400:kworker.0 00000100:00100000:2.0:1767321339.100396:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321339.100414:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321339.100417:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985656192:0@lo:400:kworker.0 00000100:00100000:2.0:1767321339.100429:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321339.100446:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655808 00000100:00100000:2.0:1767321339.100449:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655808 00000100:00100000:2.0:1767321339.100478:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655552 00000100:00100000:2.0:1767321339.100481:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655552 00000100:00100000:2.0:1767321339.100503:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5762 00000100:00100000:2.0:1767321339.100509:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985655808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321339.100534:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b88e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985655808:12345-0@lo:400:kworker.0 Request processed in 27us (539us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321339.100538:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5762 00000100:00100000:2.0:1767321339.100548:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5763 00000100:00100000:2.0:1767321339.100551:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985655552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321339.100568:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985655552:12345-0@lo:400:kworker.0 Request processed in 17us (171us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321339.100571:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5763 00000100:00080000:2.0:1767321339.100585:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321339.100592:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985656192 00000100:00100000:2.0:1767321339.100595:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985656192 00000100:00100000:2.0:1767321339.100633:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3724 00000100:00100000:2.0:1767321339.100639:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985656192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321339.100668:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985656192:12345-0@lo:400:kworker.0 Request processed in 29us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321339.100673:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3724 00000100:00080000:2.0:1767321339.100686:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321339.100706:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985656192:0@lo:400:kworker.0 00000100:00100000:2.0:1767321339.100714:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985655552:0@lo:400:kworker.0 00000100:00100000:3.0:1767321339.100771:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985656064:0@lo:400:kworker.0 00000100:00100000:3.0:1767321339.100784:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ffd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985655808:0@lo:400:kworker.0 00000100:00100000:1.0:1767321339.101146:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985655296:0@lo:38:kworker.0 00000100:00100000:1.0:1767321339.101208:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321339.101242:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985655424:0@lo:38:kworker.0 00000100:00100000:1.0:1767321339.101266:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321339.101302:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985655680:0@lo:38:kworker.0 00000100:00100000:1.0:1767321339.101325:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321339.101343:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985655936:0@lo:38:kworker.0 00000100:00100000:1.0:1767321339.101362:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321339.101389:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655296 00000100:00100000:1.0:1767321339.101393:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655296 00000100:00100000:1.0:1767321339.101413:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655680 00000100:00100000:1.0:1767321339.101415:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655680 00000100:00100000:1.0:1767321339.101425:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655936 00000100:00100000:1.0:1767321339.101427:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655936 00000100:00100000:1.0:1767321339.101439:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6622 00000100:00100000:1.0:1767321339.101444:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985655296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321339.101462:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321339.101495:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985655296:12345-0@lo:38:kworker.0 Request processed in 50us (286us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321339.101502:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6622 00000100:00100000:1.0:1767321339.101514:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6623 00000100:00100000:1.0:1767321339.101517:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985655680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321339.101527:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321339.101550:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985655680:12345-0@lo:38:kworker.0 Request processed in 31us (224us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321339.101556:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6623 00000100:00100000:1.0:1767321339.101566:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6624 00000100:00100000:1.0:1767321339.101569:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985655936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321339.101576:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321339.101597:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985655936:12345-0@lo:38:kworker.0 Request processed in 27us (234us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321339.101602:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6624 00000100:00080000:1.0:1767321339.101646:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321339.101651:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321339.101654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985655296:0@lo:38:kworker.0 00000100:00080000:1.0:1767321339.101662:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321339.101665:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321339.101667:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985655680:0@lo:38:kworker.0 00000100:00080000:1.0:1767321339.101673:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321339.101675:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321339.101678:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3fff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985655936:0@lo:38:kworker.0 00000100:00100000:3.0:1767321339.102195:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985655424 00000100:00100000:3.0:1767321339.102201:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985655424 00000100:00100000:3.0:1767321339.102231:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2146 00000100:00100000:3.0:1767321339.102236:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985655424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321339.102248:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321339.102442:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b7c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985655424:12345-0@lo:38:kworker.0 Request processed in 204us (1174us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321339.102453:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2146 00000100:00080000:1.0:1767321339.103751:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321339.103760:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321339.103764:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985655424:0@lo:38:kworker.0 00010000:00010000:0.0:1767321339.739651:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321340.763821:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321341.788195:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321342.683596:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321342.683621:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985656448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321342.683690:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321342.683894:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985656320:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321342.683925:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321342.683974:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985656448 00000100:00100000:0.0:1767321342.683981:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985656448 00000100:00100000:0.0:1767321342.684028:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985656320 00000100:00100000:0.0:1767321342.684031:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985656320 00000100:00100000:0.0:1767321342.684051:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2948 00000100:00100000:0.0:1767321342.684059:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985656448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321342.684088:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321342.684131:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985656448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (440us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321342.684139:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2948 00000100:00100000:0.0:1767321342.684155:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2949 00000100:00100000:0.0:1767321342.684161:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985656320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321342.684212:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321342.684242:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985656320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (315us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321342.684248:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2949 00000100:00100000:0.0:1767321342.684307:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985656320:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321342.684386:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321342.684391:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321342.684475:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985656448:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321342.811404:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767321343.153369:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920667904 offset 0 length 520: 4 00000100:00100000:2.0:1767321343.153415:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321343.153458:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321343.153554:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668160 00000100:00100000:1.0:1767321343.153573:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668160 00000100:00100000:3.0:1767321343.153793:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668032 00000100:00100000:3.0:1767321343.153798:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668032 00000100:00100000:3.0:1767321343.153828:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5764 00000100:00100000:3.0:1767321343.153838:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920668160:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321343.153921:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920668160:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (506us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321343.153928:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5764 00000100:00100000:3.0:1767321343.153944:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6625 00000100:00100000:3.0:1767321343.153949:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920668032:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321343.153969:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321343.153993:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920668032:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 42us (534us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321343.154000:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6625 00000100:00080000:2.0:1767321343.154852:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321343.196007:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321318/real 1767321318] req@ffff8ce1a612c380 x1853163985650048/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321343 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321343.196042:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a612c380 x1853163985650048/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321343 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321343.196062:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321343.196099:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321343.196103:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a612c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985650048:0@lo:250:kworker.0 00000100:00080000:0.0:1767321343.196271:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321343.196277:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321343.196280:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321343.196285:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6566 00000100:00080000:0.0:1767321343.196289:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321343.196295:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321343.196328:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6300 x1853163985656576/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321343.196354:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321343.196358:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321343.196360:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321343.196364:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6587 00000100:00080000:0.0:1767321343.196366:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321343.196370:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321343.196383:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6680 x1853163985656704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321343.196392:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321343.196394:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321343.196396:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321343.196399:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6587 00000100:00080000:0.0:1767321343.196401:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321343.196403:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321343.196414:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7800 x1853163985656832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321343.196450:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321343.196454:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321343.196456:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321343.196460:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6587 00000100:00080000:0.0:1767321343.196462:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321343.196465:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321343.196478:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb100 x1853163985657088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321343.196504:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321343.196507:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321343.196509:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321343.196512:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6587 00000100:00080000:0.0:1767321343.196514:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321343.196517:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321343.196528:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a61500 x1853163985657344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321343.196604:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985657216:0@lo:400:kworker.0 00000100:00100000:0.0:1767321343.196678:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321343.196745:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321343.196753:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985657472:0@lo:400:kworker.0 00000100:00100000:0.0:1767321343.196786:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321343.196832:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657216 00000100:00100000:3.0:1767321343.196847:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985656960:0@lo:400:kworker.0 00000100:00100000:3.0:1767321343.196934:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321343.196974:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321343.196976:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657216 00000100:00100000:3.0:1767321343.196984:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985657600:0@lo:400:kworker.0 00000100:00100000:3.0:1767321343.197016:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321343.197053:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985656960 00000100:00100000:0.0:1767321343.197060:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985656960 00000100:00100000:3.0:1767321343.197069:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657472 00000100:00100000:3.0:1767321343.197079:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657472 00000100:00100000:3.0:1767321343.197127:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657600 00000100:00100000:3.0:1767321343.197131:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657600 00000100:00100000:3.0:1767321343.197154:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3725 00000100:00100000:3.0:1767321343.197187:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985657472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321343.197281:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5765 00000100:00100000:3.0:1767321343.197282:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985657472:12345-0@lo:400:kworker.0 Request processed in 114us (493us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321343.197295:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3725 00000100:00100000:0.0:1767321343.197301:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985657216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321343.197323:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3726 00000100:00100000:0.0:1767321343.197367:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985657216:12345-0@lo:400:kworker.0 Request processed in 70us (686us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321343.197376:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5765 00000100:00100000:0.0:1767321343.197397:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5766 00000100:00100000:0.0:1767321343.197404:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985656960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321343.197441:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985656960:12345-0@lo:400:kworker.0 Request processed in 39us (508us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321343.197447:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5766 00000100:00100000:0.0:1767321343.197535:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985657472:0@lo:400:kworker.0 00000100:00100000:0.0:1767321343.197559:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985657216:0@lo:400:kworker.0 00000100:00100000:3.0:1767321343.197749:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985657600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321343.197799:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985657600:12345-0@lo:400:kworker.0 Request processed in 466us (780us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321343.197810:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3726 00000100:00080000:2.0:1767321343.197819:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321343.197836:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985656576:0@lo:250:kworker.0 00000100:00100000:3.0:1767321343.197870:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985657600:0@lo:400:kworker.0 00000400:00000100:2.0:1767321343.197887:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985656576 offset 0 length 520: 4 00000100:00100000:3.0:1767321343.197889:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985656960:0@lo:400:kworker.0 00000100:00100000:2.0:1767321343.197937:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985656704:0@lo:38:kworker.0 00000100:00100000:2.0:1767321343.197966:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321343.197992:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985656832:0@lo:38:kworker.0 00000100:00100000:2.0:1767321343.198013:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321343.198034:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985657088:0@lo:38:kworker.0 00000100:00100000:2.0:1767321343.198053:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321343.198070:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985657344:0@lo:38:kworker.0 00000100:00100000:2.0:1767321343.198300:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321343.198334:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985656704 00000100:00100000:2.0:1767321343.198339:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985656704 00000100:00100000:2.0:1767321343.198361:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657088 00000100:00100000:2.0:1767321343.198364:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657088 00000100:00100000:2.0:1767321343.198374:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657344 00000100:00100000:2.0:1767321343.198376:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657344 00000100:00100000:2.0:1767321343.198385:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6626 00000100:00100000:2.0:1767321343.198391:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985656704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321343.198413:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321343.198456:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985656704:12345-0@lo:38:kworker.0 Request processed in 61us (487us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321343.198465:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6626 00000100:00100000:3.0:1767321343.198474:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985656832 00000100:00100000:3.0:1767321343.198482:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985656832 00000100:00100000:2.0:1767321343.198488:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6627 00000100:00100000:2.0:1767321343.198494:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985657088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321343.198504:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321343.198525:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2147 00000100:00100000:3.0:1767321343.198533:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985656832:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321343.198549:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a612c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985657088:12345-0@lo:38:kworker.0 Request processed in 38us (479us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321343.198558:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6627 00010000:00020000:3.0:1767321343.198566:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321343.198592:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6628 00000100:00100000:2.0:1767321343.198599:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985657344:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321343.198611:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321343.198731:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985657344:12345-0@lo:38:kworker.0 Request processed in 129us (429us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321343.198737:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6628 00000100:00100000:3.0:1767321343.199007:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985656832:12345-0@lo:38:kworker.0 Request processed in 81us (599us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321343.199021:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2147 00000100:00080000:0.0:1767321343.199070:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321343.199077:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321343.199082:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985656704:0@lo:38:kworker.0 00000100:00080000:0.0:1767321343.199413:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321343.199417:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321343.199420:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985656832:0@lo:38:kworker.0 00000100:00080000:0.0:1767321343.199428:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321343.199431:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321343.199433:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985657088:0@lo:38:kworker.0 00000100:00080000:0.0:1767321343.199440:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321343.199443:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321343.199445:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985657344:0@lo:38:kworker.0 00010000:00010000:1.0:1767321343.836428:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321344.859491:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321345.883468:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321346.907633:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321347.803541:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985657728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321347.803623:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321347.803801:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657728 00000100:00100000:3.0:1767321347.803809:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657728 00000100:00100000:3.0:1767321347.803863:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2950 00000100:00100000:3.0:1767321347.803872:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985657728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321347.803901:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321347.803943:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985657728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (320us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321347.803950:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2950 00000100:00100000:3.0:1767321347.803997:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985657728:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321347.804024:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321347.816346:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985657856:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321347.816449:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321347.816601:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657856 00000100:00100000:3.0:1767321347.816611:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657856 00000100:00100000:3.0:1767321347.816684:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2951 00000100:00100000:3.0:1767321347.816692:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985657856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321347.816728:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767321347.816823:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321347.823864:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985657856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 7163us (7404us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321347.823877:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2951 00000100:00100000:1.0:1767321347.827196:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985657856:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321347.932696:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321348.267555:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321348.267666:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321348.267798:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668288 00000100:00100000:2.0:1767321348.267808:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668288 00000100:00100000:2.0:1767321348.267842:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668416 00000100:00100000:2.0:1767321348.267844:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668416 00000100:00100000:2.0:1767321348.267876:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5767 00000100:00100000:2.0:1767321348.267887:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920668416:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321348.267964:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920668416:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 79us (296us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321348.267972:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5767 00000100:00100000:2.0:1767321348.267992:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6629 00000100:00100000:2.0:1767321348.268007:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920668288:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321348.268026:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321348.268054:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920668288:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 56us (510us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321348.268060:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6629 00000100:00080000:2.0:1767321348.268078:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321348.316306:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321348.316317:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321348.316320:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321348.316324:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321348.316327:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321348.316335:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6591 00000100:00080000:2.0:1767321348.316340:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321348.316347:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321348.316394:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a3b80 x1853163985657984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321348.316423:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321348.316426:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321348.316428:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321348.316432:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6591 00000100:00080000:2.0:1767321348.316435:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321348.316438:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321348.316460:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5b8b100 x1853163985658112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321348.316509:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321348.316513:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321348.316515:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321348.316519:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6591 00000100:00080000:2.0:1767321348.316521:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321348.316524:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321348.316536:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0e00 x1853163985658368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321348.316558:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321348.316561:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321348.316563:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321348.316566:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6591 00000100:00080000:2.0:1767321348.316568:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321348.316570:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321348.316579:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1880 x1853163985658624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321348.316744:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985657984:0@lo:38:kworker.0 00000100:00100000:0.0:1767321348.316808:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321348.316876:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985658112:0@lo:38:kworker.0 00000100:00100000:0.0:1767321348.316901:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321348.316923:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985658368:0@lo:38:kworker.0 00000100:00100000:0.0:1767321348.316943:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321348.316961:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985658624:0@lo:38:kworker.0 00000100:00100000:0.0:1767321348.316992:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321348.317039:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985657984 00000100:00100000:0.0:1767321348.317046:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985657984 00000100:00100000:0.0:1767321348.317079:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985658368 00000100:00100000:0.0:1767321348.317082:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985658368 00000100:00100000:0.0:1767321348.317095:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985658624 00000100:00100000:0.0:1767321348.317097:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985658624 00000100:00100000:0.0:1767321348.317113:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6630 00000100:00100000:0.0:1767321348.317119:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985657984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321348.317139:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321348.317211:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985657984:12345-0@lo:38:kworker.0 Request processed in 90us (402us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321348.317219:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6630 00000100:00100000:0.0:1767321348.317237:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6631 00000100:00100000:0.0:1767321348.317241:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985658368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321348.317251:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321348.317275:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985658368:12345-0@lo:38:kworker.0 Request processed in 32us (331us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321348.317281:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6631 00000100:00100000:0.0:1767321348.317303:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6632 00000100:00100000:0.0:1767321348.317308:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985658624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321348.317317:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321348.317342:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985658624:12345-0@lo:38:kworker.0 Request processed in 32us (348us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321348.317347:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6632 00000100:00100000:0.0:1767321348.317382:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985658496:0@lo:400:kworker.0 00000100:00100000:0.0:1767321348.317407:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321348.317438:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985658496 00000100:00100000:0.0:1767321348.317452:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985658496 00000100:00100000:0.0:1767321348.317490:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5768 00000100:00100000:0.0:1767321348.317497:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985658496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321348.317539:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985658496:12345-0@lo:400:kworker.0 Request processed in 42us (130us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321348.317545:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5768 00000100:00100000:0.0:1767321348.317583:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985658496:0@lo:400:kworker.0 00000100:00080000:0.0:1767321348.317612:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321348.317617:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321348.317620:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985657984:0@lo:38:kworker.0 00000100:00080000:0.0:1767321348.317629:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321348.317632:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321348.317635:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985658368:0@lo:38:kworker.0 00000100:00080000:0.0:1767321348.317642:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321348.317645:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321348.317648:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985658624:0@lo:38:kworker.0 00000100:00100000:3.0:1767321348.317969:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985658112 00000100:00100000:3.0:1767321348.317984:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985658112 00000100:00100000:3.0:1767321348.318026:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2148 00000100:00100000:3.0:1767321348.318031:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985658112:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321348.318043:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321348.318083:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985658112:12345-0@lo:38:kworker.0 Request processed in 51us (1181us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321348.318089:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2148 00000100:00080000:3.0:1767321348.318120:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321348.318157:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321348.318160:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5b8b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985658112:0@lo:38:kworker.0 00000100:00100000:3.0:1767321348.318203:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321348.318213:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985658240:0@lo:400:kworker.0 00000100:00100000:3.0:1767321348.318242:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321348.318263:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985658240 00000100:00100000:3.0:1767321348.318267:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985658240 00000100:00100000:3.0:1767321348.318305:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5769 00000100:00100000:3.0:1767321348.318311:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985658240:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321348.318342:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985658240:12345-0@lo:400:kworker.0 Request processed in 31us (99us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321348.318347:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5769 00000100:00100000:3.0:1767321348.318375:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985658240:0@lo:400:kworker.0 00000100:00080000:2.0:1767321348.318701:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321348.955932:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321349.981385:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321351.003598:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321352.034148:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321352.923610:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985658752:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321352.923683:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321352.923867:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321352.923878:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985658880:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321352.923917:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321352.923951:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985658752 00000100:00100000:2.0:1767321352.923959:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985658752 00000100:00100000:2.0:1767321352.924006:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985658880 00000100:00100000:2.0:1767321352.924009:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985658880 00000100:00100000:2.0:1767321352.924029:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2952 00000100:00100000:2.0:1767321352.924037:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985658752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321352.924065:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321352.924102:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985658752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (419us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321352.924110:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2952 00000100:00100000:2.0:1767321352.924128:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2953 00000100:00100000:2.0:1767321352.924134:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985658880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321352.924149:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321352.924205:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985658880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (287us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321352.924226:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2953 00000100:00080000:2.0:1767321352.924243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321352.924247:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321352.924279:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985658880:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321352.924671:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188632680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985658752:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321353.051396:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321353.389507:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321353.389567:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321353.389684:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668544 00000100:00100000:2.0:1767321353.389696:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668544 00000100:00100000:2.0:1767321353.389741:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668672 00000100:00100000:2.0:1767321353.389744:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668672 00000100:00100000:2.0:1767321353.389816:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5770 00000100:00100000:2.0:1767321353.389828:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920668672:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321353.389901:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920668672:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 76us (333us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321353.389920:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5770 00000100:00100000:2.0:1767321353.389940:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6633 00000100:00100000:2.0:1767321353.389945:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920668544:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321353.389967:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321353.389994:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5b8b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920668544:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (496us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321353.390001:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6633 00000100:00080000:1.0:1767321353.390569:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321353.435735:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321353.435745:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321353.435748:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321353.435752:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321353.435756:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321353.435764:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6596 00000100:00080000:2.0:1767321353.435782:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321353.435788:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321353.435828:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3100 x1853163985659008/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321353.435861:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321353.435865:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321353.435866:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321353.435870:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6596 00000100:00080000:2.0:1767321353.435872:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321353.435875:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321353.435890:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3b80 x1853163985659136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321353.435933:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321353.435937:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321353.435939:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321353.435942:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6596 00000100:00080000:2.0:1767321353.435945:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321353.435947:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321353.435962:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1880 x1853163985659392/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321353.435982:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321353.435985:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321353.435987:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321353.435989:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6596 00000100:00080000:2.0:1767321353.435992:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321353.435994:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321353.436004:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0a80 x1853163985659648/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321353.436329:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985659264:0@lo:400:kworker.0 00000100:00100000:1.0:1767321353.436406:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321353.436688:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985659008:0@lo:38:kworker.0 00000100:00100000:0.0:1767321353.436721:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321353.436758:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985659136:0@lo:38:kworker.0 00000100:00100000:0.0:1767321353.436796:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321353.436819:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985659392:0@lo:38:kworker.0 00000100:00100000:0.0:1767321353.436839:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321353.436854:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985659648:0@lo:38:kworker.0 00000100:00100000:0.0:1767321353.436872:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321353.436903:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659136 00000100:00100000:0.0:1767321353.436921:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659136 00000100:00100000:0.0:1767321353.436957:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2149 00000100:00100000:0.0:1767321353.436964:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985659136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321353.436983:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321353.437024:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985659136:12345-0@lo:38:kworker.0 Request processed in 59us (227us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321353.437031:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2149 00000100:00100000:0.0:1767321353.437058:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659264 00000100:00100000:0.0:1767321353.437062:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659264 00000100:00100000:0.0:1767321353.437096:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659008 00000100:00100000:0.0:1767321353.437099:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659008 00000100:00100000:0.0:1767321353.437111:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659392 00000100:00100000:0.0:1767321353.437114:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659392 00000100:00100000:0.0:1767321353.437121:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659648 00000100:00100000:0.0:1767321353.437123:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659648 00000100:00100000:0.0:1767321353.437133:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5771 00000100:00100000:0.0:1767321353.437139:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985659264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321353.437208:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985659264:12345-0@lo:400:kworker.0 Request processed in 68us (802us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321353.437214:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5771 00000100:00100000:0.0:1767321353.437228:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6634 00000100:00100000:0.0:1767321353.437232:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985659008:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321353.437242:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321353.437264:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985659008:12345-0@lo:38:kworker.0 Request processed in 31us (542us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321353.437284:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6634 00000100:00100000:0.0:1767321353.437296:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6635 00000100:00100000:0.0:1767321353.437300:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985659392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321353.437310:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321353.437331:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985659392:12345-0@lo:38:kworker.0 Request processed in 30us (491us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321353.437337:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6635 00000100:00100000:0.0:1767321353.437346:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6636 00000100:00100000:0.0:1767321353.437360:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985659648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321353.437368:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321353.437390:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985659648:12345-0@lo:38:kworker.0 Request processed in 29us (517us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321353.437395:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6636 00000100:00100000:0.0:1767321353.437423:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985659520:0@lo:400:kworker.0 00000100:00100000:0.0:1767321353.437460:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321353.437490:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659520 00000100:00100000:0.0:1767321353.437494:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659520 00000100:00100000:0.0:1767321353.437520:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5772 00000100:00100000:0.0:1767321353.437526:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985659520:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321353.437569:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985659520:12345-0@lo:400:kworker.0 Request processed in 43us (108us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321353.437575:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5772 00000100:00100000:0.0:1767321353.437607:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985659520:0@lo:400:kworker.0 00000100:00080000:0.0:1767321353.437631:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321353.437636:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321353.437639:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985659008:0@lo:38:kworker.0 00000100:00080000:0.0:1767321353.437646:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321353.437649:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321353.437652:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985659136:0@lo:38:kworker.0 00000100:00080000:0.0:1767321353.437658:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321353.437661:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321353.437664:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985659392:0@lo:38:kworker.0 00000100:00080000:0.0:1767321353.437670:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321353.437673:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321353.437676:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985659648:0@lo:38:kworker.0 00000100:00080000:1.0:1767321353.438362:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321353.438396:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985659264:0@lo:400:kworker.0 00010000:00010000:2.0:1767321354.076531:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321355.100817:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321356.124458:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321357.147487:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321358.043449:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321358.043472:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985659776:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321358.043540:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321358.043654:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659776 00000100:00100000:3.0:1767321358.043664:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659776 00000100:00100000:3.0:1767321358.043733:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2954 00000100:00100000:3.0:1767321358.043742:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985659776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321358.043770:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321358.043811:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985659776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (271us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321358.043822:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2954 00000100:00100000:0.0:1767321358.043912:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985659776:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321358.043982:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321358.045191:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321358.045207:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985659904:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321358.045253:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321358.045361:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985659904 00000100:00100000:1.0:1767321358.045368:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985659904 00000100:00100000:1.0:1767321358.045426:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2955 00000100:00100000:1.0:1767321358.045435:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985659904:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321358.045461:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321358.045519:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985659904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321358.045526:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2955 00000100:00100000:1.0:1767321358.045575:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985659904:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321358.045721:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767321358.173466:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321358.506872:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321358.507273:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668928 00000100:00100000:0.0:1767321358.507285:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668928 00000100:00100000:0.0:1767321358.507375:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5773 00000100:00100000:0.0:1767321358.507386:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920668928:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:2.0:1767321358.507443:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321358.507484:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920668928:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (622us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321358.507495:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5773 00000100:00100000:0.0:1767321358.508484:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321358.508578:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920668800 00000100:00100000:2.0:1767321358.508585:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920668800 00000100:00100000:2.0:1767321358.508620:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6637 00000100:00100000:2.0:1767321358.508628:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920668800:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321358.508650:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321358.508703:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920668800:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 74us (223us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321358.508709:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6637 00000100:00080000:2.0:1767321358.558363:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321358.558372:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321358.558375:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321358.558380:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321358.558384:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321358.558393:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6601 00000100:00080000:2.0:1767321358.558398:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321358.558405:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321358.558441:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1880 x1853163985660032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321358.558471:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321358.558475:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321358.558477:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321358.558480:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6601 00000100:00080000:2.0:1767321358.558483:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321358.558487:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321358.558501:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3b80 x1853163985660160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321358.558580:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985660288:0@lo:400:kworker.0 00000100:00100000:1.0:1767321358.558660:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321358.558733:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321358.558737:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321358.558739:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321358.558743:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6601 00000100:00080000:2.0:1767321358.558745:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321358.558748:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321358.558762:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0e00 x1853163985660416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321358.558786:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321358.558789:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321358.558791:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321358.558794:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6601 00000100:00080000:2.0:1767321358.558797:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321358.558800:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321358.558813:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6a00 x1853163985660672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321358.558836:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321358.558847:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985660544:0@lo:400:kworker.0 00000100:00100000:2.0:1767321358.558873:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321358.558903:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660288 00000100:00100000:2.0:1767321358.558909:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660288 00000100:00100000:2.0:1767321358.558968:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660544 00000100:00100000:2.0:1767321358.558971:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660544 00000100:00100000:2.0:1767321358.558993:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5774 00000100:00100000:2.0:1767321358.559002:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985660288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321358.559078:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985660032:0@lo:38:kworker.0 00000100:00100000:3.0:1767321358.559107:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321358.559133:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985660160:0@lo:38:kworker.0 00000100:00100000:3.0:1767321358.559197:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321358.559222:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985660416:0@lo:38:kworker.0 00000100:00100000:3.0:1767321358.559245:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321358.559270:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985660672:0@lo:38:kworker.0 00000100:00100000:3.0:1767321358.559289:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321358.559315:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660160 00000100:00100000:3.0:1767321358.559319:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660160 00000100:00100000:3.0:1767321358.559341:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2150 00000100:00100000:3.0:1767321358.559346:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985660160:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321358.559364:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321358.559399:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985660160:12345-0@lo:38:kworker.0 Request processed in 50us (200us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321358.559405:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2150 00000100:00100000:3.0:1767321358.559427:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660032 00000100:00100000:3.0:1767321358.559430:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660032 00000100:00100000:3.0:1767321358.559442:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660416 00000100:00100000:3.0:1767321358.559444:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660416 00000100:00100000:3.0:1767321358.559451:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660672 00000100:00100000:3.0:1767321358.559453:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660672 00000100:00100000:3.0:1767321358.559461:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5775 00000100:00100000:3.0:1767321358.559468:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985660544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321358.559502:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985660544:12345-0@lo:400:kworker.0 Request processed in 36us (629us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321358.559508:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5775 00000100:00100000:3.0:1767321358.559522:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6638 00000100:00100000:3.0:1767321358.559526:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985660032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321358.559534:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321358.559557:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985660032:12345-0@lo:38:kworker.0 Request processed in 30us (449us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321358.559563:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6638 00000100:00100000:3.0:1767321358.559572:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6639 00000100:00100000:3.0:1767321358.559576:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985660416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321358.559584:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321358.559604:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985660416:12345-0@lo:38:kworker.0 Request processed in 26us (357us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321358.559609:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6639 00000100:00100000:3.0:1767321358.559619:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6640 00000100:00100000:3.0:1767321358.559623:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985660672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321358.559632:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321358.559653:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b2dc3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985660672:12345-0@lo:38:kworker.0 Request processed in 28us (363us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321358.559658:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6640 00000100:00100000:3.0:1767321358.559692:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985660544:0@lo:400:kworker.0 00000100:00080000:3.0:1767321358.559720:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321358.559724:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321358.559727:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985660032:0@lo:38:kworker.0 00000100:00080000:3.0:1767321358.559738:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321358.559741:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321358.559744:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985660160:0@lo:38:kworker.0 00000100:00080000:3.0:1767321358.559751:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321358.559754:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321358.559756:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985660416:0@lo:38:kworker.0 00000100:00080000:3.0:1767321358.559763:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321358.559766:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321358.559768:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985660672:0@lo:38:kworker.0 00000100:00080000:3.0:1767321358.559808:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321358.563881:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985660288:12345-0@lo:400:kworker.0 Request processed in 4870us (5211us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321358.563900:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5774 00000100:00100000:1.0:1767321358.565018:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985660288:0@lo:400:kworker.0 00010000:00010000:1.0:1767321359.195843:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321360.219535:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321361.243515:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321362.267703:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321363.163413:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321363.163432:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985660800:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321363.163505:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321363.163745:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321363.163756:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985660928:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321363.163784:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321363.163977:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660800 00000100:00100000:2.0:1767321363.163986:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660800 00000100:00100000:2.0:1767321363.164049:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985660928 00000100:00100000:2.0:1767321363.164053:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985660928 00000100:00100000:2.0:1767321363.164076:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2956 00000100:00100000:2.0:1767321363.164084:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985660800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321363.164114:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321363.164154:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985660800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (650us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321363.164161:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2956 00000100:00080000:3.0:1767321363.164210:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321363.164215:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321363.164236:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2957 00000100:00100000:2.0:1767321363.164242:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985660928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321363.164258:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321363.164296:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985660928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 54us (511us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321363.164303:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2957 00000100:00100000:0.0:1767321363.164394:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985660800:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321363.164564:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985660928:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321363.291417:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321363.627781:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321363.627838:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321363.628012:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920669056 00000100:00100000:0.0:1767321363.628026:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920669056 00000100:00100000:0.0:1767321363.628078:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920669184 00000100:00100000:0.0:1767321363.628081:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920669184 00000100:00100000:0.0:1767321363.628149:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5776 00000100:00100000:0.0:1767321363.628229:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920669184:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321363.628326:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920669184:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 166us (487us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321363.628335:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5776 00000100:00100000:0.0:1767321363.628357:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6641 00000100:00100000:0.0:1767321363.628362:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920669056:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321363.628386:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321363.628417:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920669056:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 53us (651us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321363.628423:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6641 00000100:00080000:0.0:1767321363.628441:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321363.675699:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321363.675711:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321363.675714:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321363.675718:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321363.675723:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321363.675731:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6606 00000100:00080000:0.0:1767321363.675735:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321363.675742:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321363.675795:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0000 x1853163985661056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321363.675846:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985661056:0@lo:38:kworker.0 00000100:00100000:3.0:1767321363.675938:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321363.676117:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661056 00000100:00100000:1.0:1767321363.676134:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661056 00000100:00100000:1.0:1767321363.676230:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6642 00000100:00100000:1.0:1767321363.676239:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985661056:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321363.676272:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321363.676621:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321363.676626:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321363.676629:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321363.676633:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6606 00000100:00080000:0.0:1767321363.676635:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321363.676639:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321363.676658:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1c00 x1853163985661184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321363.676978:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985661184:0@lo:38:kworker.0 00000100:00100000:3.0:1767321363.677095:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321363.677370:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985661056:12345-0@lo:38:kworker.0 Request processed in 1131us (1431us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321363.677379:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6642 00000100:00080000:0.0:1767321363.677473:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321363.677476:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321363.677479:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321363.677483:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6606 00000100:00080000:0.0:1767321363.677486:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321363.677500:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321363.677518:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1880 x1853163985661440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321363.677548:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321363.677551:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321363.677552:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321363.677556:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6606 00000100:00080000:0.0:1767321363.677558:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321363.677562:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321363.677572:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3800 x1853163985661696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321363.677599:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321363.677610:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985661568:0@lo:400:kworker.0 00000100:00100000:0.0:1767321363.677676:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321363.677719:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321363.677725:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985661312:0@lo:400:kworker.0 00000100:00100000:0.0:1767321363.677743:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321363.677768:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661568 00000100:00100000:0.0:1767321363.677774:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661568 00000100:00100000:0.0:1767321363.677814:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661312 00000100:00100000:0.0:1767321363.677816:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661312 00000100:00100000:0.0:1767321363.677829:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5777 00000100:00100000:0.0:1767321363.677834:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985661568:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321363.677882:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985661568:12345-0@lo:400:kworker.0 Request processed in 47us (205us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321363.677887:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5777 00000100:00100000:0.0:1767321363.677898:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5778 00000100:00100000:0.0:1767321363.677902:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985661312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321363.677922:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985661312:12345-0@lo:400:kworker.0 Request processed in 20us (177us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321363.677928:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5778 00000100:00080000:0.0:1767321363.677946:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321363.677984:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985661312:0@lo:400:kworker.0 00000100:00100000:0.0:1767321363.678027:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985661568:0@lo:400:kworker.0 00000100:00100000:2.0:1767321363.678684:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661184 00000100:00100000:2.0:1767321363.678691:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661184 00000100:00100000:2.0:1767321363.678724:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2151 00000100:00100000:2.0:1767321363.678730:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985661184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321363.678753:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321363.678799:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985661184:12345-0@lo:38:kworker.0 Request processed in 68us (1704us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321363.678810:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2151 00000100:00080000:3.0:1767321363.679086:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321363.679093:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321363.679097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985661056:0@lo:38:kworker.0 00000100:00100000:3.0:1767321363.679274:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985661440:0@lo:38:kworker.0 00000100:00100000:3.0:1767321363.679314:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321363.679386:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985661696:0@lo:38:kworker.0 00000100:00100000:3.0:1767321363.679411:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321363.679485:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321363.679660:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321363.679664:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985661184:0@lo:38:kworker.0 00000100:00100000:1.0:1767321363.679714:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661440 00000100:00100000:1.0:1767321363.679720:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661440 00000100:00100000:1.0:1767321363.679758:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661696 00000100:00100000:1.0:1767321363.679762:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661696 00000100:00100000:1.0:1767321363.679788:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6643 00000100:00100000:1.0:1767321363.679793:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985661440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321363.679810:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321363.679848:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985661440:12345-0@lo:38:kworker.0 Request processed in 54us (534us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321363.679856:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6643 00000100:00100000:1.0:1767321363.679868:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6644 00000100:00100000:1.0:1767321363.679872:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985661696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321363.679882:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321363.683005:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985661696:12345-0@lo:38:kworker.0 Request processed in 3120us (3582us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767321363.683010:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321363.683027:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321363.683033:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6644 00000100:00100000:3.0:1767321363.683038:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985661440:0@lo:38:kworker.0 00000100:00080000:3.0:1767321363.683067:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321363.683072:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321363.683078:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985661696:0@lo:38:kworker.0 00010000:00010000:0.0:1767321364.316016:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321365.340090:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321366.363710:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321367.391428:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321367.739855:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:0.0:1767321367.739916:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920669312 offset 0 length 520: 4 00000100:00100000:0.0:1767321367.739946:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321367.739964:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920669568 00000100:00100000:2.0:1767321367.739979:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920669568 00000100:00100000:2.0:1767321367.740071:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920669440 00000100:00100000:2.0:1767321367.740076:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920669440 00000100:00100000:2.0:1767321367.740117:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5779 00000100:00100000:2.0:1767321367.740130:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920669568:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321367.740271:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920669568:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 144us (434us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321367.740280:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5779 00000100:00100000:2.0:1767321367.740302:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6645 00000100:00100000:2.0:1767321367.740306:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920669440:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321367.740332:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321367.740360:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920669440:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (413us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321367.740367:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6645 00000100:00080000:0.0:1767321367.740660:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321368.294747:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985661824:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321368.294849:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321368.295151:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661824 00000100:00100000:1.0:1767321368.295182:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661824 00000100:00100000:1.0:1767321368.295280:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2958 00000100:00100000:1.0:1767321368.295291:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985661824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321368.295334:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321368.295389:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985661824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (540us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321368.295398:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2958 00000100:00100000:1.0:1767321368.295829:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985661952:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321368.295867:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321368.295957:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985661952 00000100:00100000:3.0:1767321368.295964:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985661952 00000100:00100000:3.0:1767321368.296004:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2959 00000100:00100000:3.0:1767321368.296012:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985661952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321368.296030:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321368.296073:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985661952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (205us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321368.296088:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2959 00000100:00100000:1.0:1767321368.296191:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985661952:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321368.296673:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985661824:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321368.296696:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321368.296701:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321368.416808:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321368.795350:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321343/real 1767321343] req@ffff8ce1a61f6300 x1853163985656576/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321368 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321368.795413:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61f6300 x1853163985656576/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321368 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321368.795448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321368.795479:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321368.795484:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985656576:0@lo:250:kworker.0 00000100:00080000:2.0:1767321368.795598:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321368.795607:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767321368.795611:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321368.795616:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6591 00000100:00080000:2.0:1767321368.795622:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321368.795631:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321368.795684:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6a00 x1853163985662080/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321368.795722:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321368.795727:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321368.795729:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321368.795732:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6611 00000100:00080000:2.0:1767321368.795735:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321368.795738:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321368.795760:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4380 x1853163985662208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321368.795768:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321368.795772:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321368.795773:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321368.795777:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6611 00000100:00080000:2.0:1767321368.795779:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321368.795783:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321368.795795:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5500 x1853163985662336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321368.795847:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321368.795853:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321368.795855:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321368.795859:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6611 00000100:00080000:2.0:1767321368.795862:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321368.795866:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321368.795879:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75880 x1853163985662592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321368.795914:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321368.795918:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321368.795920:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321368.795924:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6611 00000100:00080000:2.0:1767321368.795927:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321368.795930:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321368.795942:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985662848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321368.796032:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985662720:0@lo:400:kworker.0 00000100:00100000:2.0:1767321368.796137:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321368.796253:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321368.796260:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985662976:0@lo:400:kworker.0 00000100:00100000:2.0:1767321368.796291:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321368.796400:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985662464:0@lo:400:kworker.0 00000100:00100000:3.0:1767321368.796434:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321368.796464:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321368.796471:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985663104:0@lo:400:kworker.0 00000100:00100000:3.0:1767321368.796499:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321368.796534:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985662976 00000100:00100000:3.0:1767321368.796544:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985662976 00000100:00100000:3.0:1767321368.796600:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985663104 00000100:00100000:3.0:1767321368.796604:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985663104 00000100:00100000:3.0:1767321368.796628:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3727 00000100:00100000:3.0:1767321368.796638:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985662976:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321368.796699:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985662976:12345-0@lo:400:kworker.0 Request processed in 61us (406us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321368.796708:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3727 00000100:00100000:3.0:1767321368.796726:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3728 00000100:00100000:3.0:1767321368.796733:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985663104:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321368.796765:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985663104:12345-0@lo:400:kworker.0 Request processed in 32us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321368.796771:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3728 00000100:00100000:3.0:1767321368.796815:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985663104:0@lo:400:kworker.0 00000100:00100000:3.0:1767321368.796841:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985662976:0@lo:400:kworker.0 00000100:00100000:2.0:1767321368.796909:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985662720 00000100:00100000:2.0:1767321368.796913:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985662720 00000100:00100000:2.0:1767321368.796964:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985662464 00000100:00100000:2.0:1767321368.796967:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985662464 00000100:00100000:2.0:1767321368.796990:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5780 00000100:00100000:2.0:1767321368.796995:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985662720:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321368.797033:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985662720:12345-0@lo:400:kworker.0 Request processed in 37us (896us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321368.797039:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5780 00000100:00100000:2.0:1767321368.797053:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5781 00000100:00100000:2.0:1767321368.797059:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985662464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321368.797105:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985662464:12345-0@lo:400:kworker.0 Request processed in 46us (671us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321368.797111:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5781 00000100:00100000:2.0:1767321368.797141:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985662464:0@lo:400:kworker.0 00000100:00100000:2.0:1767321368.797161:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985662720:0@lo:400:kworker.0 00000100:00100000:1.0:1767321368.797281:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985662080:0@lo:250:kworker.0 00000400:00000100:1.0:1767321368.797315:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985662080 offset 0 length 520: 4 00000100:00100000:1.0:1767321368.797342:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985662208:0@lo:38:kworker.0 00000100:00100000:1.0:1767321368.797365:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321368.797393:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985662336:0@lo:38:kworker.0 00000100:00100000:1.0:1767321368.797420:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321368.797436:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985662592:0@lo:38:kworker.0 00000100:00100000:1.0:1767321368.797457:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321368.797474:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985662848:0@lo:38:kworker.0 00000100:00100000:1.0:1767321368.797495:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321368.797525:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985662208 00000100:00100000:1.0:1767321368.797530:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985662208 00000100:00100000:1.0:1767321368.797551:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985662592 00000100:00100000:1.0:1767321368.797553:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985662592 00000100:00100000:1.0:1767321368.797566:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985662848 00000100:00100000:1.0:1767321368.797569:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985662848 00000100:00100000:0.0:1767321368.797580:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985662336 00000100:00100000:0.0:1767321368.797585:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985662336 00000100:00100000:1.0:1767321368.797586:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6646 00000100:00100000:1.0:1767321368.797591:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985662208:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321368.797611:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2152 00000100:00100000:0.0:1767321368.797617:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985662336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321368.797617:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767321368.797626:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321368.797657:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985662336:12345-0@lo:38:kworker.0 Request processed in 41us (236us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321368.797664:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2152 00000100:00100000:2.0:1767321368.797686:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6647 00000100:00100000:2.0:1767321368.797691:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985662592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321368.797705:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321368.797753:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985662208:12345-0@lo:38:kworker.0 Request processed in 161us (387us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321368.797759:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6646 00000100:00100000:1.0:1767321368.797772:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6648 00000100:00100000:1.0:1767321368.797775:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985662848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321368.797784:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321368.797807:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985662848:12345-0@lo:38:kworker.0 Request processed in 30us (311us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321368.797813:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6648 00000100:00080000:3.0:1767321368.797900:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321368.799669:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321368.799688:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321368.799694:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985662208:0@lo:38:kworker.0 00000100:00080000:0.0:1767321368.799725:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321368.799730:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321368.799733:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985662336:0@lo:38:kworker.0 00000100:00100000:2.0:1767321368.801508:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985662592:12345-0@lo:38:kworker.0 Request processed in 3802us (4035us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321368.801527:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6647 00000100:00080000:0.0:1767321368.803995:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321368.804013:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321368.804019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985662592:0@lo:38:kworker.0 00000100:00080000:0.0:1767321368.804035:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321368.804038:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321368.804041:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985662848:0@lo:38:kworker.0 00010000:00010000:2.0:1767321369.435900:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321370.471108:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321371.487297:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321372.507504:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321372.844626:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321372.844667:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321372.844858:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920669824 00000100:00100000:3.0:1767321372.844873:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920669824 00000100:00100000:3.0:1767321372.844955:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920669696 00000100:00100000:3.0:1767321372.844957:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920669696 00000100:00100000:3.0:1767321372.844977:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5782 00000100:00100000:3.0:1767321372.844985:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920669824:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321372.845033:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6649 00000100:00100000:1.0:1767321372.845042:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920669696:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321372.845077:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321372.845098:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920669824:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 112us (485us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321372.845113:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5782 00000100:00100000:1.0:1767321372.845135:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920669696:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 90us (463us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321372.845143:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6649 00000100:00080000:0.0:1767321372.845842:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321373.404649:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985663232:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321373.404739:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321373.406228:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985663232 00000100:00100000:2.0:1767321373.406240:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985663232 00000100:00100000:2.0:1767321373.406330:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2960 00000100:00100000:2.0:1767321373.406341:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985663232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321373.406374:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321373.406444:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985663232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (1704us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321373.406451:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2960 00000100:00100000:2.0:1767321373.406507:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985663232:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321373.408692:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321373.408752:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321373.408774:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985663360:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321373.408849:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.408960:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985663360 00000100:00100000:0.0:1767321373.408967:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985663360 00000100:00100000:0.0:1767321373.409043:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2961 00000100:00100000:0.0:1767321373.409052:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985663360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321373.409083:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321373.409216:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985663360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321373.409225:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2961 00000100:00080000:3.0:1767321373.409352:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321373.409421:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985663360:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321373.531751:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321373.915324:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321373.915336:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321373.915342:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321373.915346:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321373.915351:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321373.915359:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6617 00000100:00080000:3.0:1767321373.915368:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321373.915376:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321373.915422:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22d80 x1853163985663488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321373.915447:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321373.915450:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321373.915452:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321373.915456:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6617 00000100:00080000:3.0:1767321373.915458:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321373.915462:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321373.915479:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21f80 x1853163985663616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321373.915557:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985663744:0@lo:400:kworker.0 00000100:00100000:1.0:1767321373.915629:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321373.915729:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985663744 00000100:00100000:2.0:1767321373.915738:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985663744 00000100:00100000:2.0:1767321373.915802:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5783 00000100:00100000:2.0:1767321373.915815:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985663744:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321373.915881:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321373.915885:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321373.915887:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321373.915891:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6617 00000100:00080000:3.0:1767321373.915894:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321373.915897:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321373.917488:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985663872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321373.917541:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321373.917544:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321373.917547:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321373.917551:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6617 00000100:00080000:3.0:1767321373.917554:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321373.917559:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321373.917570:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6d80 x1853163985664128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321373.918809:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985663488:0@lo:38:kworker.0 00000100:00100000:0.0:1767321373.918915:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.918968:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985663616:0@lo:38:kworker.0 00000100:00100000:0.0:1767321373.919034:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.919058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985663872:0@lo:38:kworker.0 00000100:00100000:0.0:1767321373.919079:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.919096:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985664128:0@lo:38:kworker.0 00000100:00100000:0.0:1767321373.919119:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.919187:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985663488 00000100:00100000:0.0:1767321373.919199:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985663488 00000100:00100000:0.0:1767321373.919236:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985663872 00000100:00100000:0.0:1767321373.919239:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985663872 00000100:00100000:0.0:1767321373.919248:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664128 00000100:00100000:0.0:1767321373.919249:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664128 00000100:00100000:0.0:1767321373.919262:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6650 00000100:00100000:0.0:1767321373.919270:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985663488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321373.919378:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321373.919434:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985663488:12345-0@lo:38:kworker.0 Request processed in 161us (517us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321373.919443:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6650 00000100:00100000:0.0:1767321373.919463:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6651 00000100:00100000:0.0:1767321373.919467:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985663872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321373.919476:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321373.919499:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985663872:12345-0@lo:38:kworker.0 Request processed in 31us (420us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321373.919504:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6651 00000100:00100000:0.0:1767321373.919515:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6652 00000100:00100000:0.0:1767321373.919519:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985664128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321373.919527:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321373.919546:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985664128:12345-0@lo:38:kworker.0 Request processed in 26us (426us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321373.919551:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6652 00000100:00100000:0.0:1767321373.919568:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985663616 00000100:00100000:0.0:1767321373.919572:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985663616 00000100:00100000:0.0:1767321373.919591:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2153 00000100:00100000:0.0:1767321373.919595:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985663616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321373.919604:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321373.919629:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985663616:12345-0@lo:38:kworker.0 Request processed in 32us (594us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321373.919635:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2153 00000100:00080000:0.0:1767321373.919687:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321373.919693:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321373.919697:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985663488:0@lo:38:kworker.0 00000100:00080000:0.0:1767321373.919708:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321373.919711:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321373.919714:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985663616:0@lo:38:kworker.0 00000100:00080000:0.0:1767321373.919722:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321373.919726:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321373.919728:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985663872:0@lo:38:kworker.0 00000100:00080000:0.0:1767321373.919735:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321373.919738:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321373.919741:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985664128:0@lo:38:kworker.0 00000100:00100000:0.0:1767321373.919911:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985664000:0@lo:400:kworker.0 00000100:00100000:0.0:1767321373.919940:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.919963:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321373.919969:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985664256:0@lo:400:kworker.0 00000100:00100000:0.0:1767321373.920006:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.920033:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664000 00000100:00100000:0.0:1767321373.920036:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664000 00000100:00100000:0.0:1767321373.920073:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5784 00000100:00100000:0.0:1767321373.920079:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985664000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321373.920129:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985664000:12345-0@lo:400:kworker.0 Request processed in 50us (188us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321373.920135:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5784 00000100:00100000:0.0:1767321373.920155:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664256 00000100:00100000:0.0:1767321373.920159:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664256 00000100:00100000:0.0:1767321373.920220:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3729 00000100:00100000:0.0:1767321373.920227:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985664256:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321373.920257:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985664256:12345-0@lo:400:kworker.0 Request processed in 30us (250us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321373.920264:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3729 00000100:00100000:0.0:1767321373.920298:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985664256:0@lo:400:kworker.0 00000100:00100000:0.0:1767321373.921654:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985664000:0@lo:400:kworker.0 00000100:00080000:0.0:1767321373.921744:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321373.921770:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985664384:0@lo:400:kworker.0 00000100:00100000:0.0:1767321373.921846:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321373.921897:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664384 00000100:00100000:0.0:1767321373.921904:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664384 00000100:00100000:0.0:1767321373.921942:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3730 00000100:00100000:0.0:1767321373.921950:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985664384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321373.922008:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985664384:12345-0@lo:400:kworker.0 Request processed in 59us (162us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321373.922786:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3730 00000100:00100000:0.0:1767321373.922843:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985664384:0@lo:400:kworker.0 00000100:00100000:2.0:1767321373.922946:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985663744:12345-0@lo:400:kworker.0 Request processed in 7134us (7315us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321373.922955:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5783 00000100:00100000:1.0:1767321373.923050:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985663744:0@lo:400:kworker.0 00010000:00010000:1.0:1767321374.555420:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321375.580135:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321376.604658:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321377.631522:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321377.979653:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321377.979706:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321377.980203:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920670080 00000100:00100000:0.0:1767321377.980220:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920670080 00000100:00080000:3.0:1767321377.980467:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321377.980513:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920669952 00000100:00100000:0.0:1767321377.980518:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920669952 00000100:00100000:1.0:1767321377.985493:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5785 00000100:00100000:1.0:1767321377.985531:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920670080:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321377.985677:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920670080:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 167us (6037us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321377.985687:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5785 00000100:00100000:1.0:1767321377.985751:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6653 00000100:00100000:1.0:1767321377.985758:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920669952:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321377.986494:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321377.986548:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920669952:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 787us (6839us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321377.986558:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6653 00000100:00100000:2.0:1767321378.524591:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985664640:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321378.524699:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321378.524928:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664640 00000100:00100000:3.0:1767321378.524939:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664640 00000100:00100000:3.0:1767321378.525033:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2962 00000100:00100000:3.0:1767321378.525044:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985664640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321378.525083:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321378.525135:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985664640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (437us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321378.525143:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2962 00000100:00100000:1.0:1767321378.526022:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985664512:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321378.526107:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321378.527854:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664512 00000100:00100000:3.0:1767321378.527869:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664512 00000100:00100000:3.0:1767321378.527962:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2963 00000100:00100000:3.0:1767321378.527972:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985664512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321378.528006:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321378.528097:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985664512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 127us (1989us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321378.528105:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2963 00000100:00100000:0.0:1767321378.528212:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985664512:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321378.528602:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321378.528609:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321378.528645:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985664640:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321378.651493:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321379.036051:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321379.036067:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321379.036071:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321379.036076:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321379.036081:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321379.036092:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6622 00000100:00080000:0.0:1767321379.036099:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321379.036108:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321379.036188:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18e680 x1853163985664768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321379.036537:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985664768:0@lo:38:kworker.0 00000100:00100000:2.0:1767321379.036634:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321379.036765:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321379.036770:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321379.036772:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321379.036776:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6622 00000100:00080000:0.0:1767321379.036779:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321379.036784:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321379.036803:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18c380 x1853163985664896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321379.036845:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321379.036849:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321379.036851:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321379.036855:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6622 00000100:00080000:0.0:1767321379.036858:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321379.036861:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321379.036875:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985665152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321379.036917:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321379.036921:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321379.036923:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321379.036927:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6622 00000100:00080000:0.0:1767321379.036929:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321379.036932:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321379.036944:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22d80 x1853163985665408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321379.036988:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985665280:0@lo:400:kworker.0 00000100:00100000:0.0:1767321379.037020:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321379.037081:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985664896:0@lo:38:kworker.0 00000100:00100000:2.0:1767321379.037110:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321379.037273:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664768 00000100:00100000:3.0:1767321379.037284:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664768 00000100:00100000:3.0:1767321379.037326:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665280 00000100:00100000:3.0:1767321379.037330:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665280 00000100:00100000:3.0:1767321379.037396:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5786 00000100:00100000:3.0:1767321379.037407:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985665280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321379.037466:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985665280:12345-0@lo:400:kworker.0 Request processed in 62us (444us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321379.037474:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5786 00000100:00100000:3.0:1767321379.037494:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6654 00000100:00100000:3.0:1767321379.037498:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985664768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321379.037521:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321379.037551:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985664768:12345-0@lo:38:kworker.0 Request processed in 50us (916us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321379.037557:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6654 00000100:00080000:3.0:1767321379.037578:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321379.037614:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985665664:0@lo:400:kworker.0 00000100:00100000:3.0:1767321379.037645:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321379.044945:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985665024:0@lo:400:kworker.0 00000100:00100000:1.0:1767321379.045067:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321379.045115:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985665152:0@lo:38:kworker.0 00000100:00100000:1.0:1767321379.045151:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665024 00000100:00100000:2.0:1767321379.045152:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321379.045161:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665024 00000100:00100000:2.0:1767321379.045186:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985665408:0@lo:38:kworker.0 00000100:00100000:2.0:1767321379.045208:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321379.045249:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321379.045256:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321379.045260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985664768:0@lo:38:kworker.0 00000100:00100000:2.0:1767321379.045316:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665152 00000100:00100000:2.0:1767321379.045320:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665152 00000100:00100000:2.0:1767321379.045370:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665408 00000100:00100000:2.0:1767321379.045374:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665408 00000100:00100000:2.0:1767321379.045397:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6655 00000100:00100000:2.0:1767321379.045405:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985665152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321379.045435:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321379.045485:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985665152:12345-0@lo:38:kworker.0 Request processed in 78us (331us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321379.045493:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6655 00000100:00100000:2.0:1767321379.045513:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6656 00000100:00100000:2.0:1767321379.045516:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985665408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321379.045525:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321379.045548:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985665408:12345-0@lo:38:kworker.0 Request processed in 30us (339us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321379.045553:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6656 00000100:00080000:2.0:1767321379.045640:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321379.045645:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321379.045648:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985665152:0@lo:38:kworker.0 00000100:00080000:2.0:1767321379.045657:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321379.045660:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321379.045663:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985665408:0@lo:38:kworker.0 00000100:00100000:2.0:1767321379.045720:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665664 00000100:00100000:2.0:1767321379.045724:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665664 00000100:00100000:2.0:1767321379.045760:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3731 00000100:00100000:2.0:1767321379.045767:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985665664:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321379.045811:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985665664:12345-0@lo:400:kworker.0 Request processed in 45us (8165us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321379.045818:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3731 00000100:00100000:2.0:1767321379.045842:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985664896 00000100:00100000:2.0:1767321379.045847:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985664896 00000100:00100000:2.0:1767321379.045869:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2154 00000100:00100000:2.0:1767321379.045873:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985664896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321379.045897:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321379.045930:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985664896:12345-0@lo:38:kworker.0 Request processed in 54us (8818us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321379.045936:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2154 00000100:00080000:2.0:1767321379.045960:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321379.045965:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321379.045968:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985664896:0@lo:38:kworker.0 00000100:00100000:2.0:1767321379.045990:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985665536:0@lo:400:kworker.0 00000100:00100000:2.0:1767321379.046017:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321379.046047:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665536 00000100:00100000:2.0:1767321379.046050:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665536 00000100:00100000:2.0:1767321379.046072:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3732 00000100:00100000:2.0:1767321379.046078:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985665536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321379.046108:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985665536:12345-0@lo:400:kworker.0 Request processed in 30us (90us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321379.046114:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3732 00000100:00100000:2.0:1767321379.046139:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985665536:0@lo:400:kworker.0 00000100:00080000:2.0:1767321379.049452:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321379.049648:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5787 00000100:00100000:1.0:1767321379.049665:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985665024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321379.050040:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985665024:12345-0@lo:400:kworker.0 Request processed in 380us (4972us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321379.050050:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5787 00000100:00100000:1.0:1767321379.050116:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985665024:0@lo:400:kworker.0 00000100:00100000:0.0:1767321379.053487:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985665280:0@lo:400:kworker.0 00000100:00100000:3.0:1767321379.053487:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985665664:0@lo:400:kworker.0 00010000:00010000:1.0:1767321379.675508:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321380.699572:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321381.735942:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321382.747593:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321383.091274:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321383.091343:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321383.091478:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920670208 00000100:00100000:1.0:1767321383.091492:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920670208 00000100:00100000:1.0:1767321383.091565:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920670336 00000100:00100000:1.0:1767321383.091570:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920670336 00000100:00100000:3.0:1767321383.091587:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6657 00000100:00100000:3.0:1767321383.091602:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920670208:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321383.091633:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321383.091637:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5788 00000100:00100000:1.0:1767321383.091647:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920670336:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321383.091731:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920670208:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 129us (472us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321383.091739:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6657 00000100:00080000:0.0:1767321383.091748:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321383.104110:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920670336:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 12440us (12743us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321383.104140:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5788 00000100:00100000:1.0:1767321383.643416:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985665792:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321383.643719:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321383.643840:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665792 00000100:00100000:0.0:1767321383.643863:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665792 00000100:00100000:2.0:1767321383.643955:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321383.643971:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985665920:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321383.644003:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2964 00000100:00100000:2.0:1767321383.644012:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321383.644018:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985665792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321383.644063:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321383.644106:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985665920 00000100:00100000:3.0:1767321383.644113:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985665920 00000100:00100000:3.0:1767321383.644156:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2965 00000100:00100000:3.0:1767321383.644191:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985665920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321383.644213:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767321383.644295:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321383.644299:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321383.644328:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985665920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (241us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321383.644337:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2965 00000100:00100000:2.0:1767321383.644371:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985665920:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321383.644655:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985665792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 640us (933us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321383.644664:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2964 00000100:00100000:1.0:1767321383.644668:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985665792:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321383.772412:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321384.155362:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321384.155378:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321384.155381:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321384.155386:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321384.155401:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321384.155419:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6627 00000100:00080000:0.0:1767321384.155425:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321384.155435:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321384.155503:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0000 x1853163985666048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321384.155525:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321384.155529:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321384.155531:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321384.155535:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6627 00000100:00080000:0.0:1767321384.155537:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321384.155542:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321384.155556:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1500 x1853163985666176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321384.155611:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321384.155615:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321384.155617:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321384.155621:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6627 00000100:00080000:0.0:1767321384.155624:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321384.155628:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321384.155675:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e2300 x1853163985666432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321384.155701:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321384.155704:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321384.155706:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321384.155709:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6627 00000100:00080000:0.0:1767321384.155712:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321384.155715:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321384.155728:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e2a00 x1853163985666688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321384.155814:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985666944:0@lo:400:kworker.0 00000100:00100000:0.0:1767321384.155892:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321384.155972:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321384.155978:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985666304:0@lo:400:kworker.0 00000100:00100000:0.0:1767321384.156004:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321384.156036:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985666560:0@lo:400:kworker.0 00000100:00100000:0.0:1767321384.156059:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321384.156077:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321384.156082:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985666816:0@lo:400:kworker.0 00000100:00100000:1.0:1767321384.156216:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666304 00000100:00100000:1.0:1767321384.156226:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666304 00000100:00100000:1.0:1767321384.156293:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666560 00000100:00100000:1.0:1767321384.156297:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666560 00000100:00100000:1.0:1767321384.156331:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5789 00000100:00100000:1.0:1767321384.156341:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985666304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321384.156409:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985666304:12345-0@lo:400:kworker.0 Request processed in 70us (404us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321384.156417:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5789 00000100:00100000:1.0:1767321384.156436:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5790 00000100:00100000:1.0:1767321384.156443:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985666560:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321384.156469:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985666560:12345-0@lo:400:kworker.0 Request processed in 27us (410us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321384.156475:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5790 00000100:00100000:0.0:1767321384.156546:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321384.156586:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985666560:0@lo:400:kworker.0 00000100:00100000:2.0:1767321384.156944:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666944 00000100:00100000:2.0:1767321384.156951:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666944 00000100:00100000:2.0:1767321384.156984:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666816 00000100:00100000:2.0:1767321384.156987:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666816 00000100:00100000:2.0:1767321384.157008:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3733 00000100:00100000:2.0:1767321384.157015:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985666944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321384.157050:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985666944:12345-0@lo:400:kworker.0 Request processed in 36us (1158us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321384.157057:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3733 00000100:00100000:2.0:1767321384.157070:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3734 00000100:00100000:2.0:1767321384.157076:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985666816:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321384.157120:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985666816:12345-0@lo:400:kworker.0 Request processed in 44us (573us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321384.157125:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3734 00000100:00100000:2.0:1767321384.157161:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985666048:0@lo:38:kworker.0 00000100:00100000:2.0:1767321384.157210:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321384.161771:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985666176:0@lo:38:kworker.0 00000100:00100000:2.0:1767321384.161877:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321384.161906:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985666432:0@lo:38:kworker.0 00000100:00100000:2.0:1767321384.161934:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321384.161998:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666048 00000100:00100000:1.0:1767321384.162008:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666048 00000100:00080000:3.0:1767321384.162051:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321384.162074:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6658 00000100:00100000:0.0:1767321384.162153:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985666048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321384.162198:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321384.162248:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985666048:12345-0@lo:38:kworker.0 Request processed in 163us (5037us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321384.162256:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6658 00000100:00100000:1.0:1767321384.166554:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666432 00000100:00100000:1.0:1767321384.166574:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666432 00000100:00100000:1.0:1767321384.166630:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6659 00000100:00100000:1.0:1767321384.166640:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985666432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321384.166669:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321384.166744:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985666432:12345-0@lo:38:kworker.0 Request processed in 104us (4809us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321384.166752:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6659 00000100:00100000:1.0:1767321384.166867:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985666304:0@lo:400:kworker.0 00000100:00100000:1.0:1767321384.166883:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985666944:0@lo:400:kworker.0 00000100:00100000:1.0:1767321384.166942:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666176 00000100:00100000:1.0:1767321384.166947:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666176 00000100:00100000:1.0:1767321384.166970:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2155 00000100:00100000:1.0:1767321384.166974:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985666176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321384.166984:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321384.167010:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985666176:12345-0@lo:38:kworker.0 Request processed in 35us (5134us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321384.167016:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2155 00000100:00100000:1.0:1767321384.167049:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985666816:0@lo:400:kworker.0 00000100:00100000:2.0:1767321384.168010:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985666688:0@lo:38:kworker.0 00000100:00100000:2.0:1767321384.168081:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321384.168142:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321384.168149:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321384.168153:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985666048:0@lo:38:kworker.0 00000100:00080000:2.0:1767321384.168177:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321384.168181:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321384.168185:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985666176:0@lo:38:kworker.0 00000100:00080000:2.0:1767321384.168195:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321384.168198:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321384.168201:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985666432:0@lo:38:kworker.0 00000100:00100000:1.0:1767321384.168366:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985666688 00000100:00100000:1.0:1767321384.168373:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985666688 00000100:00100000:1.0:1767321384.168419:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6660 00000100:00100000:1.0:1767321384.168426:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985666688:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321384.168443:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321384.168578:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985666688:12345-0@lo:38:kworker.0 Request processed in 150us (496us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321384.168585:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6660 00000100:00080000:2.0:1767321384.168657:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321384.168662:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321384.168665:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985666688:0@lo:38:kworker.0 00010000:00010000:2.0:1767321384.795869:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321385.819420:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321386.847591:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321387.867818:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321388.203883:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321388.204081:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321388.212771:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920670592 00000100:00100000:0.0:1767321388.212870:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920670592 00000100:00080000:3.0:1767321388.213202:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321388.213314:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920670464 00000100:00100000:0.0:1767321388.213320:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920670464 00000100:00100000:2.0:1767321388.213421:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5791 00000100:00100000:2.0:1767321388.213438:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920670592:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321388.213555:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920670592:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 122us (9689us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321388.213564:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5791 00000100:00100000:2.0:1767321388.213589:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6661 00000100:00100000:2.0:1767321388.213593:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920670464:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321388.213621:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321388.213649:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920670464:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 55us (9569us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321388.213660:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6661 00000100:00100000:1.0:1767321388.763400:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321388.763422:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985667072:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321388.763470:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321388.763482:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985667200:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321388.763543:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321388.763607:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667072 00000100:00100000:2.0:1767321388.763616:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667072 00000100:00100000:2.0:1767321388.763675:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2966 00000100:00100000:2.0:1767321388.763684:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb2850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985667072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321388.763738:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:0.0:1767321388.763850:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321388.764826:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321388.767968:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb2850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985667072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4276us (4416us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321388.767985:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2966 00000100:00100000:2.0:1767321388.768217:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667200 00000100:00100000:2.0:1767321388.768226:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667200 00000100:00100000:1.0:1767321388.768373:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985667072:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321388.769801:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321388.769945:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2967 00000100:00100000:0.0:1767321388.769965:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985667200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321388.769999:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321388.770126:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985667200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 170us (5303us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321388.770134:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2967 00000100:00100000:3.0:1767321388.770346:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985667200:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767321388.892130:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321389.275357:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321389.275378:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321389.275380:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321389.275383:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321389.275387:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321389.275394:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6632 00000100:00080000:2.0:1767321389.275398:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321389.275403:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321389.275438:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9f480 x1853163985667328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321389.275451:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321389.275453:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321389.275455:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321389.275457:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6632 00000100:00080000:2.0:1767321389.275459:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321389.275461:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321389.275479:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9e300 x1853163985667456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321389.275538:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985667584:0@lo:400:kworker.0 00000100:00100000:3.0:1767321389.275590:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321389.275699:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985667328:0@lo:38:kworker.0 00000100:00100000:0.0:1767321389.275730:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321389.275756:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985667456:0@lo:38:kworker.0 00000100:00100000:0.0:1767321389.275779:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321389.275824:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667584 00000100:00100000:0.0:1767321389.275831:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667584 00000100:00100000:0.0:1767321389.275878:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667328 00000100:00100000:0.0:1767321389.275882:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667328 00000100:00100000:0.0:1767321389.275897:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5792 00000100:00100000:0.0:1767321389.275906:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985667584:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321389.275924:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321389.275928:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321389.275930:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321389.275934:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6632 00000100:00080000:2.0:1767321389.275937:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321389.275940:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321389.275954:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9ca80 x1853163985667712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321389.275983:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321389.275987:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321389.275988:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321389.275992:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6632 00000100:00080000:2.0:1767321389.275994:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321389.275997:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321389.275998:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6662 00000100:00100000:1.0:1767321389.276004:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985667328:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321389.276007:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3f9d500 x1853163985667968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00020000:1.0:1767321389.276022:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321389.276041:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321389.276056:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985667328:12345-0@lo:38:kworker.0 Request processed in 51us (325us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321389.276062:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667456 00000100:00100000:1.0:1767321389.276063:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6662 00000100:00100000:2.0:1767321389.276065:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667456 00000100:00100000:2.0:1767321389.276086:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2156 00000100:00100000:2.0:1767321389.276091:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985667456:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321389.276099:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985667840:0@lo:400:kworker.0 00010000:00020000:2.0:1767321389.276103:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321389.276131:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985667456:12345-0@lo:38:kworker.0 Request processed in 38us (350us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321389.276135:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321389.276137:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2156 00000100:00100000:2.0:1767321389.276158:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667840 00000100:00100000:2.0:1767321389.276161:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667840 00000100:00100000:1.0:1767321389.276198:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985667712:0@lo:38:kworker.0 00000100:00100000:2.0:1767321389.276207:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5793 00000100:00100000:2.0:1767321389.276213:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985667840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321389.276220:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321389.276235:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985667968:0@lo:38:kworker.0 00000100:00100000:2.0:1767321389.276243:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985667840:12345-0@lo:400:kworker.0 Request processed in 30us (108us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321389.276249:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5793 00000100:00100000:1.0:1767321389.276255:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321389.276261:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667712 00000100:00100000:2.0:1767321389.276264:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667712 00000100:00100000:2.0:1767321389.276277:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985667968 00000100:00100000:2.0:1767321389.276279:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985667968 00000100:00080000:1.0:1767321389.276281:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321389.276286:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321389.276287:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6663 00000100:00100000:1.0:1767321389.276289:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985667328:0@lo:38:kworker.0 00000100:00100000:2.0:1767321389.276291:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985667712:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767321389.276298:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321389.276301:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:2.0:1767321389.276301:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321389.276303:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985667456:0@lo:38:kworker.0 00000100:00100000:1.0:1767321389.276334:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6664 00000100:00100000:1.0:1767321389.276338:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985667968:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321389.276342:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985667712:12345-0@lo:38:kworker.0 Request processed in 50us (121us total) trans 0 rc -19/-19 00010000:00020000:1.0:1767321389.276348:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321389.276348:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6663 00000100:00100000:1.0:1767321389.276371:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985667968:12345-0@lo:38:kworker.0 Request processed in 32us (115us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321389.276377:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6664 00000100:00100000:2.0:1767321389.276390:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985667840:0@lo:400:kworker.0 00000100:00080000:2.0:1767321389.276403:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321389.276407:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321389.276411:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321389.276414:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985667712:0@lo:38:kworker.0 00000100:00080000:1.0:1767321389.276421:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321389.276423:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321389.276426:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985667968:0@lo:38:kworker.0 00000100:00100000:0.0:1767321389.276709:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985667584:12345-0@lo:400:kworker.0 Request processed in 802us (1115us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321389.276717:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5792 00000100:00100000:3.0:1767321389.288356:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985667584:0@lo:400:kworker.0 00010000:00010000:1.0:1767321389.915659:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321390.942764:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321391.963662:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321392.299479:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321392.299606:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920670976 00000100:00100000:1.0:1767321392.299619:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920670976 00000100:00100000:1.0:1767321392.299729:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5794 00000100:00100000:1.0:1767321392.299742:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920670976:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321392.299850:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920670976:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321392.299859:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5794 00000100:00080000:2.0:1767321392.299950:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767321392.300717:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920670720 offset 0 length 520: 4 00000100:00100000:0.0:1767321392.300750:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321392.300815:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920670848 00000100:00100000:3.0:1767321392.300823:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920670848 00000100:00100000:3.0:1767321392.300862:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6665 00000100:00100000:3.0:1767321392.300869:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920670848:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321392.300894:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321392.300953:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920670848:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 82us (202us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321392.300962:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6665 00010000:00010000:1.0:1767321392.987699:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321393.371330:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321368/real 1767321368] req@ffff8ce1b3ff6a00 x1853163985662080/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321393 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321393.371373:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b3ff6a00 x1853163985662080/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321393 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321393.371389:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321393.371419:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321393.371423:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985662080:0@lo:250:kworker.0 00000100:00080000:0.0:1767321393.371581:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321393.371588:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321393.371591:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321393.371597:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6617 00000100:00080000:0.0:1767321393.371602:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321393.371610:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321393.371693:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e0000 x1853163985668096/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321393.371729:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321393.371733:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321393.371735:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321393.371739:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6637 00000100:00080000:0.0:1767321393.371742:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321393.371746:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321393.371764:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e1880 x1853163985668224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321393.371772:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321393.371776:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321393.371777:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321393.371781:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6637 00000100:00080000:0.0:1767321393.371784:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321393.371787:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321393.371805:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e3100 x1853163985668352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321393.371852:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321393.371856:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321393.371858:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321393.371861:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6637 00000100:00080000:0.0:1767321393.371864:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321393.371868:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321393.371881:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e0380 x1853163985668608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321393.371888:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985668480:0@lo:400:kworker.0 00000100:00080000:0.0:1767321393.371925:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321393.371929:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321393.371931:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321393.371936:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6637 00000100:00080000:0.0:1767321393.371939:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321393.371943:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321393.371958:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e2a00 x1853163985668864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321393.371989:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321393.372004:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985668992:0@lo:400:kworker.0 00000100:00100000:2.0:1767321393.372056:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321393.372064:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985669120:0@lo:400:kworker.0 00000100:00100000:2.0:1767321393.372089:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321393.372116:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985668736:0@lo:400:kworker.0 00000100:00100000:2.0:1767321393.372136:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321393.372191:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985669120 00000100:00100000:2.0:1767321393.372199:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985669120 00000100:00100000:2.0:1767321393.372268:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3735 00000100:00100000:2.0:1767321393.372279:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985669120:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321393.372352:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985668480 00000100:00100000:2.0:1767321393.372353:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985669120:12345-0@lo:400:kworker.0 Request processed in 75us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321393.372361:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985668480 00000100:00100000:2.0:1767321393.372365:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3735 00000100:00100000:1.0:1767321393.372411:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5795 00000100:00100000:2.0:1767321393.372419:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985668736 00000100:00100000:1.0:1767321393.372421:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985668480:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321393.372465:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985668480:12345-0@lo:400:kworker.0 Request processed in 46us (478us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321393.372471:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5795 00000100:00080000:1.0:1767321393.372521:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321393.372527:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321393.372851:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985668736 00000100:00100000:2.0:1767321393.372897:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5796 00000100:00100000:2.0:1767321393.372905:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985668736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321393.372929:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985668096:0@lo:250:kworker.0 00000100:00100000:2.0:1767321393.372961:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985668736:12345-0@lo:400:kworker.0 Request processed in 57us (823us total) trans 0 rc 0/0 00000400:00000100:3.0:1767321393.372967:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985668096 offset 0 length 520: 4 00000100:00100000:2.0:1767321393.372978:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5796 00000100:00100000:3.0:1767321393.373000:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985668224:0@lo:38:kworker.0 00000100:00100000:2.0:1767321393.373029:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985668736:0@lo:400:kworker.0 00000100:00100000:2.0:1767321393.373058:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985669120:0@lo:400:kworker.0 00000100:00100000:2.0:1767321393.373068:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985668480:0@lo:400:kworker.0 00000100:00100000:3.0:1767321393.373102:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321393.373128:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985668352:0@lo:38:kworker.0 00000100:00100000:3.0:1767321393.373153:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321393.373194:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985668608:0@lo:38:kworker.0 00000100:00100000:3.0:1767321393.373215:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321393.373234:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985668864:0@lo:38:kworker.0 00000100:00100000:3.0:1767321393.373254:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321393.373287:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985668352 00000100:00100000:3.0:1767321393.373291:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985668352 00000100:00100000:2.0:1767321393.374056:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2157 00000100:00100000:2.0:1767321393.374065:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985668352:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321393.374092:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321393.374150:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985668352:12345-0@lo:38:kworker.0 Request processed in 85us (996us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321393.374157:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2157 00000100:00100000:2.0:1767321393.374202:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985668224 00000100:00100000:2.0:1767321393.374207:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985668224 00000100:00100000:2.0:1767321393.374227:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985668608 00000100:00100000:2.0:1767321393.374230:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985668608 00000100:00100000:2.0:1767321393.374246:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985668864 00000100:00100000:2.0:1767321393.374248:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985668864 00000100:00100000:2.0:1767321393.374264:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6666 00000100:00100000:2.0:1767321393.374269:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985668224:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321393.374279:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321393.374306:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985668224:12345-0@lo:38:kworker.0 Request processed in 36us (1205us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321393.374312:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6666 00000100:00100000:2.0:1767321393.374324:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6667 00000100:00100000:2.0:1767321393.374328:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985668608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321393.374336:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321393.374356:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985668608:12345-0@lo:38:kworker.0 Request processed in 27us (1140us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321393.374361:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6667 00000100:00100000:2.0:1767321393.374372:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6668 00000100:00100000:2.0:1767321393.374375:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985668864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321393.374383:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321393.374417:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985668864:12345-0@lo:38:kworker.0 Request processed in 39us (1161us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321393.374424:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6668 00000100:00100000:0.0:1767321393.374475:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321393.374535:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321393.374540:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321393.374671:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985668992 00000100:00100000:1.0:1767321393.374676:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985668992 00000100:00100000:1.0:1767321393.375285:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3736 00000100:00100000:1.0:1767321393.375295:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985668992:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321393.376418:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985668224:0@lo:38:kworker.0 00000100:00080000:0.0:1767321393.376452:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321393.376457:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321393.376460:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985668352:0@lo:38:kworker.0 00000100:00080000:0.0:1767321393.376470:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321393.376473:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321393.376476:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985668608:0@lo:38:kworker.0 00000100:00080000:0.0:1767321393.376488:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321393.376491:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321393.376494:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985668864:0@lo:38:kworker.0 00000100:00100000:1.0:1767321393.376804:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985668992:12345-0@lo:400:kworker.0 Request processed in 1510us (2327us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321393.376815:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3736 00000100:00100000:1.0:1767321393.376860:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985668992:0@lo:400:kworker.0 00000100:00080000:0.0:1767321393.379243:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321393.379256:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321393.886558:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985669376:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321393.886688:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321393.886945:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985669376 00000100:00100000:2.0:1767321393.886958:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985669376 00000100:00100000:2.0:1767321393.887038:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2968 00000100:00100000:2.0:1767321393.887049:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985669376:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321393.887095:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321393.887154:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985669376:12345-0@lo:13:osp-pre-0-1.0 Request processed in 106us (466us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321393.887190:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2968 00000100:00100000:2.0:1767321393.887255:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985669376:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321393.889585:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985669248:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321393.889695:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321393.889775:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985669248 00000100:00100000:1.0:1767321393.889785:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985669248 00000100:00100000:1.0:1767321393.889831:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2969 00000100:00100000:1.0:1767321393.889838:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985669248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321393.889874:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321393.889922:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985669248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321393.889928:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2969 00000100:00100000:3.0:1767321393.890145:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985669248:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321394.015748:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321395.037573:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321396.059789:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321397.084082:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321397.424979:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321397.425133:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920671232 00000100:00100000:2.0:1767321397.425148:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920671232 00000100:00100000:2.0:1767321397.425292:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5797 00000100:00100000:2.0:1767321397.425306:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920671232:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321397.425422:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920671232:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 119us (457us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321397.425431:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5797 00000100:00080000:0.0:1767321397.427126:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321397.428599:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321397.428714:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920671104 00000100:00100000:3.0:1767321397.428727:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920671104 00000100:00100000:3.0:1767321397.428782:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6669 00000100:00100000:3.0:1767321397.428793:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920671104:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321397.428834:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321397.428907:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920671104:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 114us (318us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321397.428916:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6669 00010000:00010000:1.0:1767321398.111014:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321398.491751:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321398.491761:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321398.491764:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321398.491767:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321398.491771:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321398.491779:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6641 00000100:00080000:3.0:1767321398.491784:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321398.491791:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321398.491831:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5500 x1853163985669504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321398.491862:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321398.491865:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321398.491867:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321398.491870:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6641 00000100:00080000:3.0:1767321398.491872:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321398.491875:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321398.491886:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b5f80 x1853163985669632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321398.491936:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321398.491940:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321398.491942:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321398.491945:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6641 00000100:00080000:3.0:1767321398.491948:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321398.491951:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321398.491962:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b6a00 x1853163985669888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321398.491985:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321398.491989:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321398.491991:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321398.491994:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6641 00000100:00080000:3.0:1767321398.491996:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321398.491998:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321398.492007:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a54b4a80 x1853163985670144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321398.492049:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985670272:0@lo:400:kworker.0 00000100:00100000:3.0:1767321398.492112:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321398.492198:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321398.492204:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985670016:0@lo:400:kworker.0 00000100:00100000:3.0:1767321398.492228:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321398.492318:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985669760:0@lo:400:kworker.0 00000100:00100000:2.0:1767321398.492362:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321398.492390:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321398.492398:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985670400:0@lo:400:kworker.0 00000100:00100000:2.0:1767321398.492422:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321398.492473:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670016 00000100:00100000:2.0:1767321398.492480:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670016 00000100:00100000:2.0:1767321398.492534:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985669760 00000100:00100000:2.0:1767321398.492538:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985669760 00000100:00100000:2.0:1767321398.492564:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5798 00000100:00100000:2.0:1767321398.492573:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985670016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321398.492768:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ffce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985670016:12345-0@lo:400:kworker.0 Request processed in 196us (538us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321398.492776:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5798 00000100:00100000:2.0:1767321398.492794:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5799 00000100:00100000:2.0:1767321398.492800:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985669760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321398.492829:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985669760:12345-0@lo:400:kworker.0 Request processed in 30us (466us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321398.492834:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5799 00000100:00100000:2.0:1767321398.492873:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985669760:0@lo:400:kworker.0 00000100:00100000:2.0:1767321398.492900:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985670016:0@lo:400:kworker.0 00000100:00080000:0.0:1767321398.493833:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321398.493856:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985669504:0@lo:38:kworker.0 00000100:00100000:0.0:1767321398.493915:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321398.493955:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985669632:0@lo:38:kworker.0 00000100:00100000:0.0:1767321398.493978:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321398.494000:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985669888:0@lo:38:kworker.0 00000100:00100000:0.0:1767321398.494020:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321398.494036:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985670144:0@lo:38:kworker.0 00000100:00100000:0.0:1767321398.494055:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321398.494087:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985669504 00000100:00100000:0.0:1767321398.494093:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985669504 00000100:00100000:0.0:1767321398.494119:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985669888 00000100:00100000:0.0:1767321398.494122:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985669888 00000100:00100000:0.0:1767321398.494133:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670144 00000100:00100000:0.0:1767321398.494136:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670144 00000100:00100000:0.0:1767321398.494149:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6670 00000100:00100000:0.0:1767321398.494155:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985669504:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321398.494198:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321398.494235:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985669504:12345-0@lo:38:kworker.0 Request processed in 79us (320us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321398.494241:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6670 00000100:00100000:0.0:1767321398.494256:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6671 00000100:00100000:0.0:1767321398.494260:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985669888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321398.494269:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321398.494289:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985669888:12345-0@lo:38:kworker.0 Request processed in 27us (267us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321398.494294:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6671 00000100:00100000:0.0:1767321398.494303:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6672 00000100:00100000:0.0:1767321398.494307:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985670144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321398.494314:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321398.494347:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985670144:12345-0@lo:38:kworker.0 Request processed in 38us (290us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321398.494352:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6672 00000100:00080000:0.0:1767321398.494390:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321398.494395:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321398.494398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985669504:0@lo:38:kworker.0 00000100:00080000:0.0:1767321398.494406:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321398.494409:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321398.494411:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985669888:0@lo:38:kworker.0 00000100:00080000:0.0:1767321398.494417:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321398.494420:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321398.494422:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985670144:0@lo:38:kworker.0 00000100:00100000:3.0:1767321398.495225:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985669632 00000100:00100000:3.0:1767321398.495234:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985669632 00000100:00100000:3.0:1767321398.495275:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2158 00000100:00100000:3.0:1767321398.495282:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985669632:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321398.495295:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321398.495352:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985669632:12345-0@lo:38:kworker.0 Request processed in 70us (1372us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321398.495357:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2158 00000100:00080000:0.0:1767321398.495939:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321398.495945:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321398.495949:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985669632:0@lo:38:kworker.0 00000100:00100000:1.0:1767321398.510562:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670272 00000100:00100000:1.0:1767321398.510582:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670272 00000100:00100000:1.0:1767321398.510684:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670400 00000100:00100000:1.0:1767321398.510688:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670400 00000100:00100000:1.0:1767321398.510728:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3737 00000100:00100000:1.0:1767321398.510739:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985670272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321398.510942:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3738 00000100:00100000:3.0:1767321398.510970:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985670400:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321398.511072:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321398.511078:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321398.524695:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985670400:12345-0@lo:400:kworker.0 Request processed in 13731us (32262us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321398.524711:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3738 00000100:00100000:1.0:1767321398.527306:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985670400:0@lo:400:kworker.0 00000100:00100000:0.0:1767321398.531382:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985670272:12345-0@lo:400:kworker.0 Request processed in 20637us (39261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321398.531393:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3737 00000100:00100000:2.0:1767321398.531866:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985670272:0@lo:400:kworker.0 00000100:00100000:1.0:1767321399.003690:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985670528:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321399.003762:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321399.003881:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670528 00000100:00100000:2.0:1767321399.003889:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670528 00000100:00100000:2.0:1767321399.003942:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2970 00000100:00100000:2.0:1767321399.003951:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985670528:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321399.003982:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321399.004033:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985670528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (270us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321399.004041:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2970 00000100:00100000:0.0:1767321399.004285:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3f9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985670528:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321399.004768:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321399.004780:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985670656:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321399.004817:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321399.004886:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670656 00000100:00100000:1.0:1767321399.004891:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670656 00000100:00100000:1.0:1767321399.004929:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2971 00000100:00100000:1.0:1767321399.004936:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985670656:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321399.004955:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321399.004991:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3fff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985670656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321399.004998:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2971 00000100:00100000:3.0:1767321399.005207:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985670656:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321399.131641:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321400.155686:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321401.180132:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321402.209859:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321402.538055:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321402.538228:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920671488 00000100:00100000:3.0:1767321402.538244:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920671488 00000100:00100000:3.0:1767321402.538361:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5800 00000100:00100000:3.0:1767321402.538374:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920671488:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321402.538495:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920671488:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (456us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321402.538503:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5800 00000100:00080000:0.0:1767321402.540393:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321402.546705:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321402.553521:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920671360 00000100:00100000:1.0:1767321402.553543:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920671360 00000100:00100000:1.0:1767321402.553747:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6673 00000100:00100000:1.0:1767321402.553757:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920671360:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321402.553789:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321402.553887:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920671360:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 130us (7195us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321402.553894:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6673 00010000:00010000:2.0:1767321403.227721:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321403.611603:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321403.611617:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321403.611621:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321403.611626:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321403.611631:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321403.611641:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6646 00000100:00080000:3.0:1767321403.611647:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321403.611656:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321403.611713:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8000 x1853163985670784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321403.611738:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321403.611742:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321403.611744:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321403.611748:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6646 00000100:00080000:3.0:1767321403.611750:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321403.611754:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321403.611774:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61c5500 x1853163985670912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321403.611822:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321403.611828:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321403.611830:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321403.611833:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6646 00000100:00080000:3.0:1767321403.611836:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321403.611839:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321403.611851:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f5180 x1853163985671168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321403.611878:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321403.611881:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321403.611883:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321403.611886:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6646 00000100:00080000:3.0:1767321403.611889:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321403.611892:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321403.611901:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985671424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321403.611946:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985671296:0@lo:400:kworker.0 00000100:00100000:3.0:1767321403.612024:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321403.612093:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321403.612099:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985671552:0@lo:400:kworker.0 00000100:00100000:3.0:1767321403.612127:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321403.612221:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985671040:0@lo:400:kworker.0 00000100:00100000:2.0:1767321403.612253:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321403.612287:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321403.612293:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985671680:0@lo:400:kworker.0 00000100:00100000:2.0:1767321403.612318:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321403.612351:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671552 00000100:00100000:2.0:1767321403.612360:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671552 00000100:00100000:2.0:1767321403.612441:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671680 00000100:00100000:2.0:1767321403.612445:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671680 00000100:00100000:2.0:1767321403.612474:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3739 00000100:00100000:2.0:1767321403.612485:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985671552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321403.612545:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985671552:12345-0@lo:400:kworker.0 Request processed in 61us (415us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321403.612552:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3739 00000100:00100000:2.0:1767321403.612574:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3740 00000100:00100000:2.0:1767321403.612581:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985671680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321403.612613:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985671680:12345-0@lo:400:kworker.0 Request processed in 32us (293us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321403.612619:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3740 00000100:00080000:2.0:1767321403.612637:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321403.612642:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321403.612685:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985671680:0@lo:400:kworker.0 00000100:00100000:2.0:1767321403.612717:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985671552:0@lo:400:kworker.0 00000100:00100000:0.0:1767321403.612936:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985670784:0@lo:38:kworker.0 00000100:00100000:0.0:1767321403.612972:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321403.612996:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985670912:0@lo:38:kworker.0 00000100:00100000:0.0:1767321403.613024:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321403.613049:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985671168:0@lo:38:kworker.0 00000100:00100000:0.0:1767321403.613072:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321403.613089:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985671424:0@lo:38:kworker.0 00000100:00100000:0.0:1767321403.613110:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321403.613143:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670912 00000100:00100000:0.0:1767321403.613148:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670912 00000100:00100000:0.0:1767321403.613197:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2159 00000100:00100000:0.0:1767321403.613202:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985670912:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321403.613226:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321403.613262:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985670912:12345-0@lo:38:kworker.0 Request processed in 58us (237us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321403.613268:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2159 00000100:00100000:0.0:1767321403.613291:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671296 00000100:00100000:0.0:1767321403.613295:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671296 00000100:00100000:0.0:1767321403.613327:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671040 00000100:00100000:0.0:1767321403.613331:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671040 00000100:00100000:0.0:1767321403.613345:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985670784 00000100:00100000:0.0:1767321403.613348:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985670784 00000100:00100000:0.0:1767321403.613356:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671168 00000100:00100000:0.0:1767321403.613358:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671168 00000100:00100000:0.0:1767321403.613365:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671424 00000100:00100000:0.0:1767321403.613367:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671424 00000100:00100000:0.0:1767321403.613376:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5801 00000100:00100000:0.0:1767321403.613383:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985671296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321403.613632:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5802 00000100:00100000:3.0:1767321403.613642:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985671040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321403.613679:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3f9f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985671040:12345-0@lo:400:kworker.0 Request processed in 40us (1424us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321403.613688:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5802 00000100:00100000:3.0:1767321403.613705:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6674 00000100:00100000:3.0:1767321403.613709:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985670784:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321403.613720:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321403.613744:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985670784:12345-0@lo:38:kworker.0 Request processed in 33us (771us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321403.613749:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6674 00000100:00100000:3.0:1767321403.613760:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6675 00000100:00100000:3.0:1767321403.613764:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985671168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321403.613773:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321403.613797:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985671168:12345-0@lo:38:kworker.0 Request processed in 30us (724us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321403.613803:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6675 00000100:00100000:3.0:1767321403.613813:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6676 00000100:00100000:3.0:1767321403.613816:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985671424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321403.613824:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321403.613843:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985671424:12345-0@lo:38:kworker.0 Request processed in 25us (733us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321403.613848:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6676 00000100:00080000:3.0:1767321403.613902:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321403.613908:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321403.613912:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985670784:0@lo:38:kworker.0 00000100:00080000:3.0:1767321403.613922:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321403.613925:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321403.613928:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985670912:0@lo:38:kworker.0 00000100:00080000:3.0:1767321403.613935:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321403.613938:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321403.613941:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985671168:0@lo:38:kworker.0 00000100:00080000:3.0:1767321403.613948:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321403.613952:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321403.613955:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985671424:0@lo:38:kworker.0 00000100:00080000:3.0:1767321403.613990:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321403.614094:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985671296:12345-0@lo:400:kworker.0 Request processed in 34us (1391us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321403.614102:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5801 00000100:00100000:2.0:1767321403.614410:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985671040:0@lo:400:kworker.0 00000100:00100000:2.0:1767321403.614457:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985671296:0@lo:400:kworker.0 00000100:00100000:0.0:1767321404.124866:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321404.124895:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985671808:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321404.124997:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321404.125501:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985671936:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321404.125540:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321404.125670:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671808 00000100:00100000:0.0:1767321404.125681:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671808 00000100:00100000:0.0:1767321404.125746:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985671936 00000100:00100000:0.0:1767321404.125750:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985671936 00000100:00100000:0.0:1767321404.125776:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2972 00000100:00100000:0.0:1767321404.125787:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985671808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321404.125827:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321404.125882:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985671808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (884us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321404.125891:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2972 00000100:00100000:0.0:1767321404.125912:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2973 00000100:00100000:0.0:1767321404.125920:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985671936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321404.125942:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321404.126784:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985671808:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321404.141059:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985671936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 15122us (15501us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321404.141080:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2973 00000100:00100000:3.0:1767321404.142996:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985671936:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321404.252244:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321405.275543:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321406.302240:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321407.323490:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321407.665397:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321407.665752:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920671744 00000100:00100000:2.0:1767321407.665765:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920671744 00000100:00100000:2.0:1767321407.665855:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5803 00000100:00100000:2.0:1767321407.665869:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920671744:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767321407.665966:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321407.665983:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920671744:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 119us (605us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321407.665992:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5803 00000100:00100000:0.0:1767321407.667079:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321407.667257:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920671616 00000100:00100000:2.0:1767321407.667275:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920671616 00000100:00100000:2.0:1767321407.667312:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6677 00000100:00100000:2.0:1767321407.667319:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920671616:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321407.670221:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321407.670333:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920671616:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 3009us (3253us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321407.670888:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6677 00010000:00010000:1.0:1767321408.363545:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321408.745500:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321408.745512:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321408.745515:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321408.745519:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321408.745524:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321408.745534:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6651 00000100:00080000:1.0:1767321408.745539:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321408.745545:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321408.745596:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98a80 x1853163985672064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321408.745630:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321408.745633:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321408.745635:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321408.745637:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6651 00000100:00080000:1.0:1767321408.745639:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321408.745641:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321408.745652:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9a300 x1853163985672192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321408.745701:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321408.745706:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321408.745708:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321408.745712:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6651 00000100:00080000:1.0:1767321408.745714:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321408.745718:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321408.745732:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9aa00 x1853163985672448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321408.745774:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321408.745778:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321408.745781:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321408.745785:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6651 00000100:00080000:1.0:1767321408.745788:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767321408.745789:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767321408.745792:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321408.745807:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98e00 x1853163985672704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321408.745819:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985672320:0@lo:400:kworker.0 00000100:00100000:1.0:1767321408.745888:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985672576:0@lo:400:kworker.0 00000100:00100000:3.0:1767321408.745922:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321408.745925:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985672832:0@lo:400:kworker.0 00000100:00100000:1.0:1767321408.745967:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321408.745977:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321408.745994:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985672960:0@lo:400:kworker.0 00000100:00100000:3.0:1767321408.746016:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321408.746052:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321408.746102:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672832 00000100:00100000:3.0:1767321408.746102:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672320 00000100:00100000:2.0:1767321408.746113:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672832 00000100:00100000:3.0:1767321408.746113:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672320 00000100:00100000:3.0:1767321408.746210:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672576 00000100:00100000:0.0:1767321408.746212:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672960 00000100:00100000:3.0:1767321408.746214:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672576 00000100:00100000:0.0:1767321408.746216:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672960 00000100:00100000:2.0:1767321408.746222:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3741 00000100:00100000:1.0:1767321408.746231:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5804 00000100:00100000:2.0:1767321408.746237:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985672832:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321408.746242:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985672320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321408.746251:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3742 00000100:00100000:3.0:1767321408.746251:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5805 00000100:00100000:0.0:1767321408.746262:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985672960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321408.746262:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985672576:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321408.746323:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985672320:12345-0@lo:400:kworker.0 Request processed in 82us (400us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321408.746324:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985672960:12345-0@lo:400:kworker.0 Request processed in 64us (271us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321408.746324:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985672576:12345-0@lo:400:kworker.0 Request processed in 63us (306us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321408.746350:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3742 00000100:00100000:3.0:1767321408.746350:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5805 00000100:00100000:1.0:1767321408.746362:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5804 00000100:00080000:3.0:1767321408.746378:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321408.746384:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321408.746389:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321408.746397:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985672064:0@lo:38:kworker.0 00000100:00100000:1.0:1767321408.746435:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321408.746438:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985672576:0@lo:400:kworker.0 00000100:00100000:3.0:1767321408.746482:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985672320:0@lo:400:kworker.0 00000100:00100000:1.0:1767321408.746483:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985672192:0@lo:38:kworker.0 00000100:00100000:1.0:1767321408.746517:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321408.746522:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985672960:0@lo:400:kworker.0 00000100:00100000:1.0:1767321408.746550:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985672448:0@lo:38:kworker.0 00000100:00100000:3.0:1767321408.746555:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672192 00000100:00100000:3.0:1767321408.746560:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672192 00000100:00100000:1.0:1767321408.746576:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321408.746588:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2160 00000100:00100000:3.0:1767321408.746594:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985672192:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321408.746598:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985672704:0@lo:38:kworker.0 00010000:00020000:3.0:1767321408.746619:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321408.746620:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321408.746658:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672064 00000100:00100000:1.0:1767321408.746663:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672064 00000100:00100000:3.0:1767321408.746671:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985672192:12345-0@lo:38:kworker.0 Request processed in 75us (153us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321408.746679:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc98000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985672832:12345-0@lo:400:kworker.0 Request processed in 446us (709us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321408.746688:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6678 00000100:00100000:2.0:1767321408.746689:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3741 00000100:00100000:1.0:1767321408.746696:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985672064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321408.746709:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321408.746725:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985672832:0@lo:400:kworker.0 00000100:00100000:1.0:1767321408.746745:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc99500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985672064:12345-0@lo:38:kworker.0 Request processed in 48us (308us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321408.746758:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6678 00000100:00100000:3.0:1767321408.746758:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2160 00000100:00100000:2.0:1767321408.746775:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672448 00000100:00100000:2.0:1767321408.746780:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672448 00000100:00100000:1.0:1767321408.746781:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985672704 00000100:00100000:1.0:1767321408.746787:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985672704 00000100:00100000:2.0:1767321408.746804:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6679 00000100:00100000:2.0:1767321408.746811:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985672448:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321408.746817:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6680 00000100:00100000:1.0:1767321408.746823:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985672704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321408.746825:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767321408.746838:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321408.746862:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985672448:12345-0@lo:38:kworker.0 Request processed in 49us (286us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321408.746870:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6679 00000100:00100000:1.0:1767321408.746875:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985672704:12345-0@lo:38:kworker.0 Request processed in 50us (253us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321408.746883:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6680 00000100:00080000:1.0:1767321408.746929:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321408.746934:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321408.746938:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985672064:0@lo:38:kworker.0 00000100:00080000:1.0:1767321408.746950:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321408.746953:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321408.746957:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985672192:0@lo:38:kworker.0 00000100:00080000:1.0:1767321408.746966:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321408.746970:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321408.746973:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985672448:0@lo:38:kworker.0 00000100:00080000:1.0:1767321408.746981:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321408.746985:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321408.746988:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985672704:0@lo:38:kworker.0 00000100:00100000:3.0:1767321409.244424:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321409.244461:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985673088:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321409.244565:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321409.244677:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321409.244687:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985673216:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321409.244713:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321409.245160:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673088 00000100:00100000:1.0:1767321409.245197:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673088 00000100:00100000:2.0:1767321409.245226:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673216 00000100:00100000:2.0:1767321409.245231:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673216 00000100:00100000:1.0:1767321409.245422:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2974 00000100:00100000:2.0:1767321409.245432:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2975 00000100:00100000:1.0:1767321409.245439:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985673088:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321409.245446:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985673216:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321409.245484:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:1.0:1767321409.245487:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321409.245552:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985673216:12345-0@lo:13:osp-pre-0-1.0 Request processed in 110us (835us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321409.245556:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985673088:12345-0@lo:13:osp-pre-1-1.0 Request processed in 122us (989us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321409.245566:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2975 00000100:00100000:2.0:1767321409.245761:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985673216:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321409.245806:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985673088:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321409.249797:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2974 00010000:00010000:2.0:1767321409.371773:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321410.395438:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321411.422655:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321412.444094:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321412.780004:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321412.780236:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321412.780437:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920672000 00000100:00100000:0.0:1767321412.780450:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920672000 00000100:00100000:0.0:1767321412.780545:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920671872 00000100:00100000:0.0:1767321412.780549:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920671872 00000100:00100000:0.0:1767321412.780571:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5806 00000100:00100000:0.0:1767321412.780582:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920672000:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321412.780677:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1adc9aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920672000:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 97us (704us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321412.780686:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5806 00000100:00100000:0.0:1767321412.780709:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6681 00000100:00100000:0.0:1767321412.780714:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920671872:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321412.780740:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321412.780781:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920671872:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 65us (546us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321412.780788:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6681 00000100:00080000:3.0:1767321412.782923:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767321413.473132:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321413.851470:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321413.851484:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321413.851487:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321413.851494:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321413.851500:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321413.851511:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6657 00000100:00080000:1.0:1767321413.851517:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321413.851526:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321413.851599:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9a300 x1853163985673344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321413.851634:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321413.851638:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321413.851640:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321413.851644:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6657 00000100:00080000:1.0:1767321413.851647:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321413.851651:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321413.851664:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98a80 x1853163985673472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321413.851740:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985673600:0@lo:400:kworker.0 00000100:00100000:3.0:1767321413.851848:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321413.852005:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321413.852010:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321413.852012:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321413.852016:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6657 00000100:00080000:1.0:1767321413.852019:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321413.852022:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321413.852275:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673600 00000100:00100000:0.0:1767321413.852286:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673600 00000100:00100000:0.0:1767321413.852366:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5807 00000100:00100000:0.0:1767321413.852379:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985673600:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321413.852481:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98e00 x1853163985673728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321413.852517:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321413.852522:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321413.852524:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321413.852528:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6657 00000100:00080000:1.0:1767321413.852531:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321413.852535:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321413.852548:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e300 x1853163985673984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321413.852631:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985673856:0@lo:400:kworker.0 00000100:00100000:2.0:1767321413.852670:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321413.852706:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321413.852712:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985674112:0@lo:400:kworker.0 00000100:00100000:2.0:1767321413.852737:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321413.852768:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673856 00000100:00100000:2.0:1767321413.852772:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673856 00000100:00100000:2.0:1767321413.852820:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5808 00000100:00100000:2.0:1767321413.852827:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985673856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321413.852866:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985673856:12345-0@lo:400:kworker.0 Request processed in 38us (194us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321413.852874:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5808 00000100:00100000:2.0:1767321413.852903:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985674112 00000100:00100000:2.0:1767321413.852908:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985674112 00000100:00100000:2.0:1767321413.852938:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3743 00000100:00100000:2.0:1767321413.852945:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985674112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321413.852976:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985674112:12345-0@lo:400:kworker.0 Request processed in 31us (238us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321413.852983:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3743 00000100:00100000:2.0:1767321413.853009:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985673344:0@lo:38:kworker.0 00000100:00100000:2.0:1767321413.853050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321413.853073:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985673472:0@lo:38:kworker.0 00000100:00100000:2.0:1767321413.853103:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321413.853127:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985673728:0@lo:38:kworker.0 00000100:00100000:2.0:1767321413.853146:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321413.853184:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985673984:0@lo:38:kworker.0 00000100:00100000:2.0:1767321413.853209:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321413.853253:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673472 00000100:00100000:2.0:1767321413.853257:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673472 00000100:00100000:2.0:1767321413.853283:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2161 00000100:00100000:2.0:1767321413.853288:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985673472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321413.853314:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321413.853350:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985673472:12345-0@lo:38:kworker.0 Request processed in 60us (246us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321413.853356:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2161 00000100:00100000:2.0:1767321413.853377:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673344 00000100:00100000:2.0:1767321413.853381:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673344 00000100:00100000:2.0:1767321413.853395:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673728 00000100:00100000:2.0:1767321413.853397:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673728 00000100:00100000:2.0:1767321413.853406:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985673984 00000100:00100000:2.0:1767321413.853408:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985673984 00000100:00100000:2.0:1767321413.853418:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6682 00000100:00100000:2.0:1767321413.853423:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985673344:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321413.853432:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321413.853458:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985673344:12345-0@lo:38:kworker.0 Request processed in 34us (408us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321413.853464:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6682 00000100:00100000:2.0:1767321413.853477:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6683 00000100:00100000:2.0:1767321413.853481:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985673728:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321413.853491:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321413.853515:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985673728:12345-0@lo:38:kworker.0 Request processed in 33us (368us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321413.853521:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6683 00000100:00100000:2.0:1767321413.853532:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6684 00000100:00100000:2.0:1767321413.853536:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985673984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321413.853545:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321413.853567:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985673984:12345-0@lo:38:kworker.0 Request processed in 30us (357us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321413.853573:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6684 00000100:00100000:2.0:1767321413.853627:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985674112:0@lo:400:kworker.0 00000100:00100000:2.0:1767321413.853639:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985673856:0@lo:400:kworker.0 00000100:00080000:2.0:1767321413.853672:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321413.853677:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321413.853681:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985673344:0@lo:38:kworker.0 00000100:00080000:2.0:1767321413.853689:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321413.853692:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321413.853695:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985673472:0@lo:38:kworker.0 00000100:00080000:2.0:1767321413.853702:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321413.853705:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321413.853708:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985673728:0@lo:38:kworker.0 00000100:00080000:2.0:1767321413.853715:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321413.853718:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321413.853720:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985673984:0@lo:38:kworker.0 00000100:00100000:2.0:1767321413.853770:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985674240:0@lo:400:kworker.0 00000100:00100000:2.0:1767321413.853799:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321413.853832:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985674240 00000100:00100000:2.0:1767321413.853836:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985674240 00000100:00100000:2.0:1767321413.853867:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3744 00000100:00100000:2.0:1767321413.853873:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985674240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321413.853908:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985674240:12345-0@lo:400:kworker.0 Request processed in 36us (108us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321413.853914:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3744 00000100:00100000:2.0:1767321413.853940:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985674240:0@lo:400:kworker.0 00000100:00080000:2.0:1767321413.853983:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321413.853988:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321413.853990:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321413.856998:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985673600:12345-0@lo:400:kworker.0 Request processed in 4606us (5134us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321413.857018:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5807 00000100:00100000:3.0:1767321413.863282:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985673600:0@lo:400:kworker.0 00000100:00100000:3.0:1767321414.364007:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985674368:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321414.364118:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321414.364321:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985674368 00000100:00100000:1.0:1767321414.364332:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985674368 00000100:00100000:1.0:1767321414.364405:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2976 00000100:00100000:1.0:1767321414.364416:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985674368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321414.364458:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321414.364512:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985674368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321414.364520:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2976 00000100:00100000:2.0:1767321414.364597:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985674496:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321414.364630:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321414.364717:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985674496 00000100:00100000:1.0:1767321414.364721:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985674496 00000100:00100000:1.0:1767321414.364753:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2977 00000100:00100000:1.0:1767321414.364759:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985674496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321414.364776:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321414.364816:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985674496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 57us (186us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321414.364822:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2977 00000100:00100000:3.0:1767321414.365110:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985674368:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321414.365259:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985674496:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321414.491748:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321415.515410:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321416.539433:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321417.565211:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321417.898696:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321417.898912:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920672256 00000100:00100000:0.0:1767321417.898930:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920672256 00000100:00100000:0.0:1767321417.899039:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5809 00000100:00100000:0.0:1767321417.899052:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920672256:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321417.899157:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920672256:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 106us (476us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321417.899242:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5809 00000100:00080000:2.0:1767321417.900429:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321417.901722:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321417.901983:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920672128 00000100:00100000:3.0:1767321417.901997:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920672128 00000100:00100000:3.0:1767321417.902095:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6685 00000100:00100000:3.0:1767321417.902105:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920672128:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321417.902134:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321417.902264:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920672128:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 158us (552us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321417.902274:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6685 00000400:00000100:0.0:1767321417.903644:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920672384 offset 0 length 520: 4 00010000:00010000:2.0:1767321418.587429:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321418.971761:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321393/real 1767321393] req@ffff8ce19e8e0000 x1853163985668096/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321418 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321418.971810:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19e8e0000 x1853163985668096/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321418 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321418.971828:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321418.976126:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321418.976150:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985668096:0@lo:250:kworker.0 00000100:00080000:2.0:1767321418.983269:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321418.983284:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767321418.983290:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321418.983300:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6641 00000100:00080000:2.0:1767321418.983307:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321418.983317:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321418.985699:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12300 x1853163985674624/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321418.985746:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321418.985752:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321418.985757:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321418.985762:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6662 00000100:00080000:2.0:1767321418.985766:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321418.985773:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321418.985806:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc99f80 x1853163985674752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321418.985816:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321418.985820:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321418.985822:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321418.985825:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6662 00000100:00080000:2.0:1767321418.985827:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321418.985831:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321418.985844:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc98e00 x1853163985674880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321418.985886:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321418.985890:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321418.985892:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321418.985896:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6662 00000100:00080000:2.0:1767321418.985898:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321418.985902:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321418.985913:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1adc9a300 x1853163985675136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321418.985939:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321418.985942:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321418.985946:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321418.985949:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6662 00000100:00080000:2.0:1767321418.985952:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321418.985955:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321418.985968:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73e300 x1853163985675392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321418.986037:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985675648:0@lo:400:kworker.0 00000100:00100000:2.0:1767321418.986123:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321418.986244:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321418.986252:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985675008:0@lo:400:kworker.0 00000100:00100000:2.0:1767321418.986280:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321418.986325:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675648 00000100:00100000:2.0:1767321418.986336:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675648 00000100:00100000:3.0:1767321418.986506:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985674624:0@lo:250:kworker.0 00000400:00000100:3.0:1767321418.986551:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985674624 offset 0 length 520: 4 00000100:00100000:3.0:1767321418.986571:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985674752:0@lo:38:kworker.0 00000100:00100000:3.0:1767321418.986597:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321418.986619:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985674880:0@lo:38:kworker.0 00000100:00100000:3.0:1767321418.986646:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321418.986663:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985675136:0@lo:38:kworker.0 00000100:00100000:2.0:1767321418.986681:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3745 00000100:00100000:3.0:1767321418.986689:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321418.986692:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985675648:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321418.986761:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985675648:12345-0@lo:400:kworker.0 Request processed in 70us (637us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321418.986770:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3745 00000100:00080000:2.0:1767321418.986799:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321418.986817:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985674880 00000100:00100000:2.0:1767321418.986821:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985674880 00000100:00100000:2.0:1767321418.986843:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2162 00000100:00100000:2.0:1767321418.986849:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985674880:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321418.986874:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321418.986909:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985674880:12345-0@lo:38:kworker.0 Request processed in 58us (261us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321418.986919:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2162 00000100:00100000:2.0:1767321418.986972:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985675648:0@lo:400:kworker.0 00000100:00100000:2.0:1767321418.987535:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985675264:0@lo:400:kworker.0 00000100:00100000:2.0:1767321418.987647:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321418.991430:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985675520:0@lo:400:kworker.0 00000100:00100000:1.0:1767321418.991551:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321418.995731:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985675392:0@lo:38:kworker.0 00000100:00100000:3.0:1767321418.995849:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675520 00000100:00100000:3.0:1767321418.995859:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675520 00000100:00100000:3.0:1767321418.996735:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3746 00000100:00100000:3.0:1767321418.996753:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985675520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321418.996863:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985675520:12345-0@lo:400:kworker.0 Request processed in 116us (5310us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321418.996873:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3746 00000100:00100000:3.0:1767321418.997196:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985675520:0@lo:400:kworker.0 00000100:00080000:3.0:1767321418.997248:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321418.999400:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675008 00000100:00100000:2.0:1767321418.999420:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675008 00000100:00100000:2.0:1767321418.999497:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985674752 00000100:00100000:2.0:1767321418.999501:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985674752 00000100:00100000:2.0:1767321418.999515:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675136 00000100:00100000:2.0:1767321418.999518:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675136 00000100:00100000:2.0:1767321418.999537:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675264 00000100:00100000:2.0:1767321418.999541:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675264 00000100:00100000:2.0:1767321418.999576:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5810 00000100:00100000:2.0:1767321418.999587:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985675008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321418.999703:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985675008:12345-0@lo:400:kworker.0 Request processed in 117us (13420us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321418.999713:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5810 00000100:00100000:2.0:1767321418.999733:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5811 00000100:00100000:2.0:1767321418.999740:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985675264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321418.999777:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985675264:12345-0@lo:400:kworker.0 Request processed in 38us (12196us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321418.999783:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5811 00000100:00100000:2.0:1767321418.999796:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6686 00000100:00100000:2.0:1767321418.999800:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985674752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321418.999825:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321418.999857:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985674752:12345-0@lo:38:kworker.0 Request processed in 54us (13259us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321418.999864:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6686 00000100:00100000:2.0:1767321418.999876:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6687 00000100:00100000:2.0:1767321418.999881:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce187feb450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985675136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321418.999890:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321418.999927:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce187feb450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985675136:12345-0@lo:38:kworker.0 Request processed in 44us (13236us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321418.999933:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6687 00000100:00100000:2.0:1767321419.012210:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985675008:0@lo:400:kworker.0 00000100:00080000:2.0:1767321419.012263:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321419.012288:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985675264:0@lo:400:kworker.0 00000100:00100000:2.0:1767321419.012318:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321419.012376:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321419.012381:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321419.012384:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985674752:0@lo:38:kworker.0 00000100:00080000:2.0:1767321419.012394:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321419.012398:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321419.012401:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985674880:0@lo:38:kworker.0 00000100:00080000:2.0:1767321419.012410:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321419.012413:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321419.012416:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1adc9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985675136:0@lo:38:kworker.0 00000100:00100000:0.0:1767321419.012500:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675392 00000100:00100000:0.0:1767321419.012518:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675392 00000100:00100000:0.0:1767321419.012605:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6688 00000100:00100000:0.0:1767321419.012615:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985675392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321419.012645:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321419.012745:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985675392:12345-0@lo:38:kworker.0 Request processed in 130us (426us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321419.012754:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6688 00000100:00080000:2.0:1767321419.012831:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321419.012838:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321419.013408:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985675392:0@lo:38:kworker.0 00000100:00100000:2.0:1767321419.483580:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321419.483603:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985675776:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321419.483675:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321419.484046:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675776 00000100:00100000:3.0:1767321419.484063:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675776 00000100:00100000:3.0:1767321419.484157:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2978 00000100:00100000:3.0:1767321419.484191:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985675776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321419.484233:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321419.484351:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985675776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 162us (682us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321419.484359:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2978 00000100:00100000:0.0:1767321419.484559:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985675776:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321419.486541:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985675904:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321419.486648:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321419.486852:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985675904 00000100:00100000:0.0:1767321419.486863:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985675904 00000100:00100000:0.0:1767321419.486935:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2979 00000100:00100000:0.0:1767321419.486947:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985675904:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321419.486986:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321419.487063:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985675904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (414us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321419.487071:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2979 00000100:00100000:1.0:1767321419.487346:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985675904:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321419.611655:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321420.636050:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321421.660681:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321422.683546:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321423.021707:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321423.022560:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321423.022680:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920672640 00000100:00100000:1.0:1767321423.022698:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920672640 00000100:00100000:1.0:1767321423.022797:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920672512 00000100:00100000:1.0:1767321423.022801:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920672512 00000100:00100000:1.0:1767321423.022825:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5812 00000100:00100000:1.0:1767321423.022840:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920672640:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321423.022947:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920672640:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 111us (1257us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321423.022955:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5812 00000100:00100000:1.0:1767321423.022975:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6689 00000100:00100000:1.0:1767321423.022979:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920672512:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321423.023003:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321423.023028:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920672512:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 48us (475us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321423.023033:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6689 00000100:00080000:3.0:1767321423.024991:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321423.707597:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321424.094019:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321424.094038:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321424.094042:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321424.094058:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321424.094064:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321424.094075:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6667 00000100:00080000:0.0:1767321424.094088:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321424.094097:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321424.094358:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985676032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321424.094441:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985676032:0@lo:38:kworker.0 00000100:00100000:3.0:1767321424.094535:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321424.094814:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676032 00000100:00100000:2.0:1767321424.094838:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676032 00000100:00100000:2.0:1767321424.094900:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6690 00000100:00100000:2.0:1767321424.094910:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985676032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321424.094938:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321424.095000:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321424.095005:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321424.095008:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321424.095012:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6667 00000100:00080000:0.0:1767321424.095015:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321424.095020:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321424.095044:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4a80 x1853163985676160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321424.103708:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985676160:0@lo:38:kworker.0 00000100:00100000:2.0:1767321424.104261:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985676032:12345-0@lo:38:kworker.0 Request processed in 9334us (9709us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321424.104277:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6690 00000100:00100000:3.0:1767321424.106246:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321424.106390:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321424.106408:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321424.106413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985676032:0@lo:38:kworker.0 00000100:00100000:2.0:1767321424.107000:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676160 00000100:00100000:2.0:1767321424.107015:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676160 00000100:00100000:2.0:1767321424.107098:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2163 00000100:00100000:2.0:1767321424.107109:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985676160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321424.107140:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321424.107300:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985676160:12345-0@lo:38:kworker.0 Request processed in 191us (1065us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321424.107308:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2163 00000100:00080000:1.0:1767321424.108074:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321424.108091:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321424.108096:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985676160:0@lo:38:kworker.0 00000100:00080000:0.0:1767321424.121134:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321424.121144:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321424.121149:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321424.121159:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6667 00000100:00080000:0.0:1767321424.121195:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321424.121204:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321424.121232:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6d80 x1853163985676416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321424.121277:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321424.121280:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321424.121282:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321424.121285:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6667 00000100:00080000:0.0:1767321424.121288:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321424.121291:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321424.121300:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4e00 x1853163985676672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321424.121368:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985676800:0@lo:400:kworker.0 00000100:00100000:0.0:1767321424.121455:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321424.121508:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321424.121513:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985676544:0@lo:400:kworker.0 00000100:00100000:0.0:1767321424.121536:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321424.121572:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676800 00000100:00100000:0.0:1767321424.121582:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676800 00000100:00100000:0.0:1767321424.121659:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3747 00000100:00100000:0.0:1767321424.121669:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985676800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321424.122356:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985676800:12345-0@lo:400:kworker.0 Request processed in 675us (887us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321424.122373:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3747 00000100:00100000:1.0:1767321424.122446:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985676928:0@lo:400:kworker.0 00000100:00100000:0.0:1767321424.122464:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985676800:0@lo:400:kworker.0 00000100:00100000:1.0:1767321424.122503:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321424.122516:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676544 00000100:00100000:0.0:1767321424.122527:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676544 00000100:00100000:1.0:1767321424.122555:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321424.122567:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985676288:0@lo:400:kworker.0 00000100:00100000:1.0:1767321424.122596:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321424.122664:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985676416:0@lo:38:kworker.0 00000100:00080000:3.0:1767321424.122755:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321424.122762:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321424.122790:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676928 00000100:00100000:3.0:1767321424.122801:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676928 00000100:00100000:3.0:1767321424.122860:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3748 00000100:00100000:3.0:1767321424.122870:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985676928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321424.122920:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985676928:12345-0@lo:400:kworker.0 Request processed in 52us (419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321424.122927:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3748 00000100:00080000:3.0:1767321424.122948:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321424.125762:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676288 00000100:00100000:0.0:1767321424.125779:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676288 00000100:00100000:0.0:1767321424.125881:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5813 00000100:00100000:0.0:1767321424.125891:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985676544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321424.125968:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5814 00000100:00100000:2.0:1767321424.126001:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985676288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321424.126119:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985676288:12345-0@lo:400:kworker.0 Request processed in 139us (3522us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321424.126129:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5814 00000100:00080000:2.0:1767321424.126228:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321424.126262:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321424.126292:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985676672:0@lo:38:kworker.0 00000100:00100000:2.0:1767321424.126329:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321424.126367:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676416 00000100:00100000:2.0:1767321424.126373:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676416 00000100:00100000:2.0:1767321424.126414:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985676672 00000100:00100000:2.0:1767321424.126418:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985676672 00000100:00100000:2.0:1767321424.126435:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6691 00000100:00100000:2.0:1767321424.126440:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985676416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321424.126468:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321424.126508:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985676416:12345-0@lo:38:kworker.0 Request processed in 66us (246us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321424.126515:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6691 00000100:00100000:2.0:1767321424.126530:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6692 00000100:00100000:2.0:1767321424.126533:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985676672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321424.126542:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321424.126569:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985676672:12345-0@lo:38:kworker.0 Request processed in 33us (238us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321424.126576:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6692 00000100:00080000:2.0:1767321424.126615:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321424.126621:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321424.126625:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985676416:0@lo:38:kworker.0 00000100:00080000:2.0:1767321424.126636:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321424.126640:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321424.126643:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985676672:0@lo:38:kworker.0 00000100:00100000:0.0:1767321424.126825:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985676544:12345-0@lo:400:kworker.0 Request processed in 928us (5279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321424.126838:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5813 00000100:00100000:2.0:1767321424.127117:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985676544:0@lo:400:kworker.0 00000100:00100000:1.0:1767321424.128694:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985676928:0@lo:400:kworker.0 00000100:00100000:1.0:1767321424.128780:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985676288:0@lo:400:kworker.0 00000100:00100000:0.0:1767321424.603729:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321424.603759:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985677056:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321424.603873:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321424.604137:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321424.604150:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985677184:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321424.604213:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321424.604395:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677056 00000100:00100000:0.0:1767321424.604405:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677056 00000100:00100000:0.0:1767321424.604469:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2980 00000100:00100000:0.0:1767321424.604556:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985677056:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321424.604601:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321424.604660:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985677056:12345-0@lo:13:osp-pre-1-1.0 Request processed in 180us (787us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321424.604668:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2980 00000100:00100000:0.0:1767321424.604690:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677184 00000100:00100000:0.0:1767321424.604694:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677184 00000100:00100000:0.0:1767321424.604715:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2981 00000100:00100000:0.0:1767321424.604720:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985677184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321424.604737:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321424.604768:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985677184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (555us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321424.604774:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2981 00000100:00100000:2.0:1767321424.604954:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985677184:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321424.605220:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985677056:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321424.731818:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321425.759120:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321426.779928:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321427.808206:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321428.156021:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321428.156082:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321428.156231:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920672896 00000100:00100000:2.0:1767321428.156247:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920672896 00000100:00100000:2.0:1767321428.156355:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920672768 00000100:00100000:2.0:1767321428.156359:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920672768 00000100:00100000:2.0:1767321428.156386:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5815 00000100:00100000:2.0:1767321428.156400:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920672896:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321428.156519:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920672896:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (522us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321428.156528:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5815 00000100:00100000:2.0:1767321428.156551:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6693 00000100:00100000:2.0:1767321428.156555:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920672768:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321428.156580:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321428.156610:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920672768:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (528us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321428.156621:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6693 00000100:00080000:0.0:1767321428.157528:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321428.828197:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321429.211686:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321429.211699:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321429.211703:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321429.211708:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321429.211714:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321429.211726:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6672 00000100:00080000:3.0:1767321429.211733:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321429.211743:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321429.211814:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1c00 x1853163985677312/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321429.211888:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985677312:0@lo:38:kworker.0 00000100:00100000:2.0:1767321429.211992:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321429.212092:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321429.212096:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321429.212099:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321429.212113:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6672 00000100:00080000:3.0:1767321429.212117:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321429.212122:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321429.212139:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1180 x1853163985677440/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321429.212206:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677312 00000100:00100000:0.0:1767321429.212219:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677312 00000100:00100000:0.0:1767321429.212273:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6694 00000100:00100000:0.0:1767321429.212283:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985677312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321429.212310:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321429.223263:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985677440:0@lo:38:kworker.0 00000100:00100000:2.0:1767321429.223382:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321429.223516:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677440 00000100:00100000:1.0:1767321429.223528:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677440 00000100:00100000:1.0:1767321429.223593:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2164 00000100:00100000:1.0:1767321429.223603:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985677440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321429.223633:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321429.223687:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985677440:12345-0@lo:38:kworker.0 Request processed in 83us (305us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321429.223694:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2164 00000100:00080000:3.0:1767321429.223935:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321429.223942:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321429.223946:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321429.223962:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6672 00000100:00080000:3.0:1767321429.223967:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321429.223973:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321429.223999:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1f80 x1853163985677696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321429.224033:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321429.224037:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321429.224039:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321429.224042:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6672 00000100:00080000:3.0:1767321429.224044:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321429.224048:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321429.224059:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d0e00 x1853163985677952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321429.224107:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985678080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224148:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321429.224200:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321429.224206:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985677824:0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224233:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321429.224266:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678080 00000100:00100000:3.0:1767321429.224273:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678080 00000100:00100000:3.0:1767321429.224320:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3749 00000100:00100000:3.0:1767321429.224327:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985678080:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224377:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985678080:12345-0@lo:400:kworker.0 Request processed in 49us (227us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321429.224384:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3749 00000100:00100000:3.0:1767321429.224408:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677824 00000100:00100000:3.0:1767321429.224412:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677824 00000100:00100000:3.0:1767321429.224441:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5816 00000100:00100000:3.0:1767321429.224447:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985677824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224477:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985677824:12345-0@lo:400:kworker.0 Request processed in 30us (243us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321429.224483:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5816 00000100:00100000:3.0:1767321429.224529:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985677824:0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224540:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985678080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224597:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985678208:0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224621:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321429.224639:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321429.224643:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985677568:0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224661:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321429.224679:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678208 00000100:00100000:3.0:1767321429.224683:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678208 00000100:00100000:3.0:1767321429.224707:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3750 00000100:00100000:3.0:1767321429.224712:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985678208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224743:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985678208:12345-0@lo:400:kworker.0 Request processed in 31us (121us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321429.224749:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3750 00000100:00100000:3.0:1767321429.224764:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677568 00000100:00100000:3.0:1767321429.224767:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677568 00000100:00100000:3.0:1767321429.224789:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5817 00000100:00100000:3.0:1767321429.224795:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985677568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224822:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985677568:12345-0@lo:400:kworker.0 Request processed in 28us (160us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321429.224828:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5817 00000100:00100000:3.0:1767321429.224853:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985677568:0@lo:400:kworker.0 00000100:00100000:3.0:1767321429.224861:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985678208:0@lo:400:kworker.0 00000100:00080000:1.0:1767321429.225271:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321429.225278:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321429.225280:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321429.226933:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985677312:12345-0@lo:38:kworker.0 Request processed in 14635us (14925us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321429.226960:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6694 00000100:00080000:2.0:1767321429.229115:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321429.229134:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321429.229141:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985677312:0@lo:38:kworker.0 00000100:00100000:2.0:1767321429.229200:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985677696:0@lo:38:kworker.0 00000100:00100000:2.0:1767321429.229291:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321429.229328:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985677952:0@lo:38:kworker.0 00000100:00100000:2.0:1767321429.229349:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321429.229387:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321429.229392:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321429.229395:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985677440:0@lo:38:kworker.0 00000100:00100000:0.0:1767321429.229524:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677696 00000100:00100000:0.0:1767321429.229534:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677696 00000100:00100000:0.0:1767321429.229893:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985677952 00000100:00100000:0.0:1767321429.229898:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985677952 00000100:00100000:0.0:1767321429.229928:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6695 00000100:00100000:0.0:1767321429.229937:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985677696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321429.229973:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321429.230036:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985677696:12345-0@lo:38:kworker.0 Request processed in 98us (745us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321429.230044:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6695 00000100:00100000:0.0:1767321429.230064:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6696 00000100:00100000:0.0:1767321429.230068:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985677952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321429.230078:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321429.230105:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985677952:12345-0@lo:38:kworker.0 Request processed in 35us (754us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321429.230111:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6696 00000100:00080000:2.0:1767321429.230261:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321429.230267:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321429.230271:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985677696:0@lo:38:kworker.0 00000100:00080000:2.0:1767321429.230281:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321429.230285:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321429.230288:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985677952:0@lo:38:kworker.0 00000100:00100000:3.0:1767321429.723661:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321429.723704:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985678336:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321429.723814:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321429.723995:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678336 00000100:00100000:1.0:1767321429.724007:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678336 00000100:00100000:1.0:1767321429.724093:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2982 00000100:00100000:1.0:1767321429.724106:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985678336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321429.724153:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321429.724251:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985678336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 147us (437us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321429.724261:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2982 00000100:00100000:0.0:1767321429.726130:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985678336:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321429.730887:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321429.730924:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985678464:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321429.731054:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321429.731217:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678464 00000100:00100000:3.0:1767321429.731231:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678464 00000100:00100000:3.0:1767321429.731304:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2983 00000100:00100000:3.0:1767321429.731315:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985678464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321429.731370:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321429.731433:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985678464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 119us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321429.731442:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2983 00000100:00100000:2.0:1767321429.731787:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985678464:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321429.854453:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321430.876062:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321431.899646:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321432.923612:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321433.261937:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321433.262609:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920673152 00000100:00100000:3.0:1767321433.262625:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920673152 00000100:00100000:3.0:1767321433.262818:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5818 00000100:00100000:3.0:1767321433.262833:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920673152:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321433.262969:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920673152:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 139us (1045us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321433.262978:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5818 00000100:00100000:2.0:1767321433.267115:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321433.267227:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920673024 00000100:00100000:0.0:1767321433.267241:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920673024 00000100:00100000:0.0:1767321433.267305:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6697 00000100:00100000:0.0:1767321433.267313:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920673024:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321433.267341:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321433.267431:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920673024:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 116us (329us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321433.267439:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6697 00000100:00080000:1.0:1767321433.272667:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321433.949562:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321434.331410:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321434.331425:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321434.331428:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321434.331432:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321434.331438:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321434.331449:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6677 00000100:00080000:1.0:1767321434.331456:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321434.331465:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321434.331534:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0e00 x1853163985678592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321434.331572:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321434.331576:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321434.331578:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321434.331582:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6677 00000100:00080000:1.0:1767321434.331585:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321434.331588:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321434.331612:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff3b80 x1853163985678720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321434.331668:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321434.331674:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321434.331676:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321434.331680:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6677 00000100:00080000:1.0:1767321434.331683:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321434.331688:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321434.331702:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1f80 x1853163985678976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321434.331731:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321434.331734:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321434.331735:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321434.331739:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6677 00000100:00080000:1.0:1767321434.331741:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321434.331745:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321434.331754:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0700 x1853163985679232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321434.331946:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985679488:0@lo:400:kworker.0 00000100:00100000:0.0:1767321434.332058:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321434.332209:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321434.332216:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985678848:0@lo:400:kworker.0 00000100:00100000:0.0:1767321434.332247:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321434.332283:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985679104:0@lo:400:kworker.0 00000100:00100000:0.0:1767321434.332307:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321434.332328:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321434.332333:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985679360:0@lo:400:kworker.0 00000100:00100000:0.0:1767321434.332357:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321434.332393:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678848 00000100:00100000:0.0:1767321434.332404:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678848 00000100:00100000:0.0:1767321434.332480:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985679104 00000100:00100000:0.0:1767321434.332483:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985679104 00000100:00100000:0.0:1767321434.332512:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5819 00000100:00100000:0.0:1767321434.332521:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985678848:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321434.332585:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985678848:12345-0@lo:400:kworker.0 Request processed in 65us (336us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321434.332593:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5819 00000100:00100000:0.0:1767321434.332625:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5820 00000100:00100000:0.0:1767321434.332633:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985679104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321434.332664:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985679104:12345-0@lo:400:kworker.0 Request processed in 32us (356us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321434.332670:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5820 00000100:00080000:0.0:1767321434.332688:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321434.332733:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985679104:0@lo:400:kworker.0 00000100:00100000:0.0:1767321434.332761:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985678848:0@lo:400:kworker.0 00000100:00100000:3.0:1767321434.333122:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985679488 00000100:00100000:3.0:1767321434.333129:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985679488 00000100:00100000:3.0:1767321434.333203:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985679360 00000100:00100000:3.0:1767321434.333206:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985679360 00000100:00100000:3.0:1767321434.333231:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3751 00000100:00100000:3.0:1767321434.333238:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985679488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321434.333292:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985679488:12345-0@lo:400:kworker.0 Request processed in 54us (1234us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321434.333299:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3751 00000100:00100000:3.0:1767321434.333312:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3752 00000100:00100000:3.0:1767321434.333318:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985679360:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321434.333364:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985679360:12345-0@lo:400:kworker.0 Request processed in 46us (1005us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321434.333369:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3752 00000100:00100000:3.0:1767321434.333405:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985679360:0@lo:400:kworker.0 00000100:00100000:3.0:1767321434.333432:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985679488:0@lo:400:kworker.0 00000100:00100000:2.0:1767321434.333838:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985678592:0@lo:38:kworker.0 00000100:00100000:2.0:1767321434.333899:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321434.333941:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985678720:0@lo:38:kworker.0 00000100:00100000:2.0:1767321434.333967:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321434.333988:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985678976:0@lo:38:kworker.0 00000100:00100000:2.0:1767321434.334030:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321434.334050:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985679232:0@lo:38:kworker.0 00000100:00100000:2.0:1767321434.334074:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321434.334112:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678592 00000100:00100000:2.0:1767321434.334117:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678592 00000100:00100000:2.0:1767321434.334146:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678976 00000100:00100000:2.0:1767321434.334149:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678976 00000100:00100000:2.0:1767321434.334184:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985679232 00000100:00100000:2.0:1767321434.334187:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985679232 00000100:00100000:2.0:1767321434.334205:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6698 00000100:00100000:2.0:1767321434.334212:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985678592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321434.334237:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321434.334274:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985678592:12345-0@lo:38:kworker.0 Request processed in 62us (376us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321434.334281:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6698 00000100:00100000:2.0:1767321434.334297:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6699 00000100:00100000:2.0:1767321434.334301:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985678976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321434.334311:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321434.334333:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188632a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985678976:12345-0@lo:38:kworker.0 Request processed in 30us (302us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321434.334339:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6699 00000100:00100000:2.0:1767321434.334350:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6700 00000100:00100000:2.0:1767321434.334354:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985679232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321434.334362:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321434.334383:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985679232:12345-0@lo:38:kworker.0 Request processed in 28us (309us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321434.334388:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6700 00000100:00080000:2.0:1767321434.334431:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321434.334436:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321434.334440:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985678592:0@lo:38:kworker.0 00000100:00080000:2.0:1767321434.334448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321434.334452:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321434.334454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985678976:0@lo:38:kworker.0 00000100:00080000:2.0:1767321434.334460:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321434.334463:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321434.334466:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985679232:0@lo:38:kworker.0 00000100:00100000:1.0:1767321434.335114:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985678720 00000100:00100000:1.0:1767321434.335121:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985678720 00000100:00100000:1.0:1767321434.335198:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2165 00000100:00100000:1.0:1767321434.335205:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985678720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321434.335220:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321434.335269:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985678720:12345-0@lo:38:kworker.0 Request processed in 64us (1301us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321434.335275:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2165 00000100:00080000:0.0:1767321434.335379:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321434.335385:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321434.335972:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321434.335981:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321434.335985:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985678720:0@lo:38:kworker.0 00000100:00100000:2.0:1767321434.846027:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985679616:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321434.846154:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321434.846538:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985679616 00000100:00100000:3.0:1767321434.846554:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985679616 00000100:00100000:3.0:1767321434.846636:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2984 00000100:00100000:3.0:1767321434.846648:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985679616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321434.846692:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321434.846765:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985679616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 119us (611us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321434.846775:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2984 00000100:00100000:0.0:1767321434.846896:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985679616:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321434.847194:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321434.847210:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985679744:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321434.847255:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321434.847426:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985679744 00000100:00100000:1.0:1767321434.847434:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985679744 00000100:00100000:1.0:1767321434.847479:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2985 00000100:00100000:1.0:1767321434.847488:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985679744:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321434.847509:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321434.847551:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985679744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (295us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321434.847558:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2985 00000100:00100000:1.0:1767321434.847589:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985679744:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321434.973313:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321435.995616:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321437.019498:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321438.043416:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321438.380080:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321438.380230:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920673408 00000100:00100000:3.0:1767321438.380247:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920673408 00000100:00100000:2.0:1767321438.380327:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00080000:1.0:1767321438.380481:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321438.384897:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920673280 00000100:00100000:3.0:1767321438.384919:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920673280 00000100:00100000:3.0:1767321438.385013:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5821 00000100:00100000:3.0:1767321438.385029:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920673408:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321438.385137:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920673408:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 115us (5073us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321438.385145:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5821 00000100:00100000:3.0:1767321438.385195:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6701 00000100:00100000:3.0:1767321438.385200:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920673280:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321438.385221:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321438.385249:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920673280:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 47us (4924us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321438.385255:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6701 00010000:00010000:1.0:1767321439.067656:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321439.454242:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321439.454253:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321439.454256:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321439.454260:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321439.454265:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321439.454275:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6682 00000100:00080000:0.0:1767321439.454281:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321439.454288:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321439.454337:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5f80 x1853163985679872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321439.454358:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321439.454361:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321439.454363:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321439.454366:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6682 00000100:00080000:0.0:1767321439.454369:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321439.454373:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321439.454384:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4e00 x1853163985680000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321439.454421:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321439.454424:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321439.454426:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321439.454429:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6682 00000100:00080000:0.0:1767321439.454431:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321439.454434:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321439.454446:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff4a80 x1853163985680256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321439.454470:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321439.454473:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321439.454474:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321439.454477:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6682 00000100:00080000:0.0:1767321439.454480:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321439.454482:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321439.454493:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb480 x1853163985680512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321439.454547:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985680640:0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.454670:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985680768:0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.454739:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321439.454842:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321439.454849:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985680128:0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.454874:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321439.454911:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680768 00000100:00100000:3.0:1767321439.454919:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680768 00000100:00100000:3.0:1767321439.454973:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3753 00000100:00100000:3.0:1767321439.454982:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985680768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.455033:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985680768:12345-0@lo:400:kworker.0 Request processed in 52us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321439.455041:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3753 00000100:00080000:3.0:1767321439.455065:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321439.455081:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985680384:0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.455106:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321439.455149:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985680768:0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.455213:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680128 00000100:00100000:3.0:1767321439.455217:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680128 00000100:00100000:3.0:1767321439.455245:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680384 00000100:00100000:3.0:1767321439.455248:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680384 00000100:00100000:3.0:1767321439.455267:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5822 00000100:00100000:3.0:1767321439.455273:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985680128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.455308:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985680128:12345-0@lo:400:kworker.0 Request processed in 35us (433us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321439.455314:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5822 00000100:00100000:3.0:1767321439.455327:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5823 00000100:00100000:3.0:1767321439.455332:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985680384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.455360:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985680384:12345-0@lo:400:kworker.0 Request processed in 27us (253us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321439.455365:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5823 00000100:00080000:3.0:1767321439.455385:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321439.455400:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985680384:0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.455421:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985680128:0@lo:400:kworker.0 00000100:00100000:0.0:1767321439.455761:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321439.455894:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985679872:0@lo:38:kworker.0 00000100:00100000:2.0:1767321439.455924:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321439.455948:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985680000:0@lo:38:kworker.0 00000100:00100000:2.0:1767321439.455968:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321439.456003:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985680256:0@lo:38:kworker.0 00000100:00100000:2.0:1767321439.456023:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321439.456040:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985680512:0@lo:38:kworker.0 00000100:00100000:2.0:1767321439.456059:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321439.456086:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985679872 00000100:00100000:2.0:1767321439.456092:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985679872 00000100:00100000:2.0:1767321439.456114:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680256 00000100:00100000:2.0:1767321439.456116:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680256 00000100:00100000:2.0:1767321439.456128:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680512 00000100:00100000:2.0:1767321439.456131:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680512 00000100:00100000:2.0:1767321439.456144:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6702 00000100:00100000:2.0:1767321439.456149:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985679872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321439.456208:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321439.456247:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985679872:12345-0@lo:38:kworker.0 Request processed in 96us (321us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321439.456254:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6702 00000100:00100000:2.0:1767321439.456270:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6703 00000100:00100000:2.0:1767321439.456274:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985680256:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321439.456286:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321439.456308:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985680256:12345-0@lo:38:kworker.0 Request processed in 32us (284us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321439.456314:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6703 00000100:00100000:2.0:1767321439.456325:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6704 00000100:00100000:2.0:1767321439.456329:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985680512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321439.456338:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321439.456358:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985680512:12345-0@lo:38:kworker.0 Request processed in 27us (298us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321439.456363:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6704 00000100:00080000:2.0:1767321439.456411:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321439.456416:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321439.456419:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985679872:0@lo:38:kworker.0 00000100:00080000:2.0:1767321439.456427:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321439.456429:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321439.456432:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985680256:0@lo:38:kworker.0 00000100:00080000:2.0:1767321439.456439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321439.456442:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321439.456445:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985680512:0@lo:38:kworker.0 00000100:00100000:0.0:1767321439.462000:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680000 00000100:00100000:0.0:1767321439.462017:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680000 00000100:00100000:0.0:1767321439.462078:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2166 00000100:00100000:0.0:1767321439.462098:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985680000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321439.462122:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321439.462265:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985680000:12345-0@lo:38:kworker.0 Request processed in 177us (6294us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321439.462272:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2166 00000100:00100000:3.0:1767321439.462412:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680640 00000100:00100000:3.0:1767321439.462419:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680640 00000100:00100000:3.0:1767321439.462488:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3754 00000100:00100000:3.0:1767321439.462496:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985680640:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321439.462582:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985680640:12345-0@lo:400:kworker.0 Request processed in 87us (6821us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321439.462589:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3754 00000100:00080000:0.0:1767321439.462919:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321439.462967:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985680640:0@lo:400:kworker.0 00000100:00080000:2.0:1767321439.463537:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321439.463546:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321439.463552:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985680000:0@lo:38:kworker.0 00000100:00100000:3.0:1767321439.963977:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321439.964001:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985680896:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321439.964070:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321439.964527:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985680896 00000100:00100000:2.0:1767321439.964536:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985680896 00000100:00100000:2.0:1767321439.964594:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2986 00000100:00100000:2.0:1767321439.964602:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985680896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321439.964632:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321439.964677:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985680896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (607us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321439.964685:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2986 00000100:00100000:0.0:1767321439.965297:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321439.965314:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985681024:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321439.965362:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321439.967202:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985680896:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321439.967718:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985681024 00000100:00100000:2.0:1767321439.967726:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985681024 00000100:00100000:2.0:1767321439.968084:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2987 00000100:00100000:2.0:1767321439.968097:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985681024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321439.968128:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321439.968255:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985681024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 161us (2891us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321439.968263:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2987 00000100:00100000:1.0:1767321439.968388:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985681024:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321440.092419:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321441.117753:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321442.139372:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321442.476213:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767321442.476247:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920673536 offset 0 length 520: 4 00000100:00100000:2.0:1767321442.476267:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321442.477246:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920673792 00000100:00100000:3.0:1767321442.477259:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920673792 00000100:00100000:3.0:1767321442.477341:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920673664 00000100:00100000:3.0:1767321442.477345:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920673664 00000100:00080000:0.0:1767321442.477351:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321442.477371:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5824 00000100:00100000:3.0:1767321442.477390:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920673792:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321442.477473:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920673792:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 84us (1268us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321442.477480:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5824 00000100:00100000:3.0:1767321442.477500:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6705 00000100:00100000:3.0:1767321442.477504:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920673664:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321442.477526:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321442.477551:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920673664:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 45us (1283us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321442.477557:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6705 00010000:00010000:2.0:1767321443.164464:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321443.547478:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321418/real 1767321418] req@ffff8ce184a12300 x1853163985674624/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321443 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321443.547513:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce184a12300 x1853163985674624/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321443 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321443.547526:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321443.547552:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321443.547556:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985674624:0@lo:250:kworker.0 00000100:00080000:0.0:1767321443.550340:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321443.550356:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321443.550361:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321443.550372:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6667 00000100:00080000:0.0:1767321443.550379:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321443.550400:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321443.550450:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdad80 x1853163985681152/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321443.550479:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321443.550483:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321443.550485:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321443.550489:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6687 00000100:00080000:0.0:1767321443.550492:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321443.550496:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321443.550516:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd8a80 x1853163985681280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321443.550527:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321443.550531:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321443.550533:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767321443.550538:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985681152:0@lo:250:kworker.0 00000100:00080000:0.0:1767321443.550539:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6687 00000100:00080000:0.0:1767321443.550554:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321443.550560:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321443.550574:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcda680 x1853163985681408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:2.0:1767321443.550615:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985681152 offset 0 length 520: 4 00000100:00080000:0.0:1767321443.550623:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321443.550628:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321443.550631:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321443.550634:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6687 00000100:00080000:0.0:1767321443.550637:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321443.550640:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321443.550656:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd8380 x1853163985681664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321443.550679:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321443.550682:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321443.550683:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321443.550686:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6687 00000100:00080000:0.0:1767321443.550698:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321443.550702:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321443.550715:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9f80 x1853163985681920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321443.550750:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985681536:0@lo:400:kworker.0 00000100:00100000:3.0:1767321443.550800:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321443.550849:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985681792:0@lo:400:kworker.0 00000100:00100000:3.0:1767321443.550875:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321443.550921:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985681536 00000100:00100000:3.0:1767321443.550930:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985681536 00000100:00100000:3.0:1767321443.551017:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985681792 00000100:00100000:3.0:1767321443.551022:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985681792 00000100:00100000:3.0:1767321443.551053:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5825 00000100:00100000:3.0:1767321443.551065:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985681536:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321443.551077:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321443.551136:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5826 00000100:00100000:3.0:1767321443.551142:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985681536:12345-0@lo:400:kworker.0 Request processed in 75us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321443.551208:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985681792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321443.551248:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5825 00000100:00100000:3.0:1767321443.551401:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985681792:0@lo:400:kworker.0 00000100:00100000:3.0:1767321443.551434:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985681536:0@lo:400:kworker.0 00000100:00100000:2.0:1767321443.551493:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985681280:0@lo:38:kworker.0 00000100:00100000:2.0:1767321443.551535:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321443.551553:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985681408:0@lo:38:kworker.0 00000100:00100000:2.0:1767321443.551575:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321443.551591:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985681664:0@lo:38:kworker.0 00000100:00100000:2.0:1767321443.551608:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321443.551624:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985681920:0@lo:38:kworker.0 00000100:00100000:2.0:1767321443.551644:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321443.551659:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985682176:0@lo:400:kworker.0 00000100:00100000:2.0:1767321443.551676:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985681408 00000100:00100000:2.0:1767321443.551681:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985681408 00000100:00100000:0.0:1767321443.551718:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321443.551722:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2167 00000100:00100000:2.0:1767321443.551729:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985681408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321443.551750:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321443.551766:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682176 00000100:00100000:0.0:1767321443.551772:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682176 00000100:00100000:2.0:1767321443.551789:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985681408:12345-0@lo:38:kworker.0 Request processed in 58us (211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321443.551796:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2167 00000100:00100000:0.0:1767321443.551819:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3755 00000100:00100000:0.0:1767321443.551829:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985682176:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321443.551831:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321443.551840:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321443.551844:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985681408:0@lo:38:kworker.0 00000100:00080000:2.0:1767321443.551869:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321443.551924:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985681280 00000100:00100000:3.0:1767321443.551927:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985681280 00000100:00100000:3.0:1767321443.551953:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985681664 00000100:00100000:3.0:1767321443.551955:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985681664 00000100:00100000:1.0:1767321443.551960:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985681792:12345-0@lo:400:kworker.0 Request processed in 164us (437us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321443.551961:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985682176:12345-0@lo:400:kworker.0 Request processed in 130us (241us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321443.551969:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985681920 00000100:00100000:3.0:1767321443.551976:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985681920 00000100:00100000:1.0:1767321443.551989:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5826 00000100:00100000:1.0:1767321443.552025:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6706 00000100:00100000:1.0:1767321443.552031:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985681280:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321443.552069:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3755 00010000:00020000:1.0:1767321443.552076:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321443.552132:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985681280:12345-0@lo:38:kworker.0 Request processed in 100us (598us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321443.552139:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6706 00000100:00100000:1.0:1767321443.552153:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6707 00000100:00100000:1.0:1767321443.552157:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985681664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321443.552188:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321443.552221:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985681664:12345-0@lo:38:kworker.0 Request processed in 62us (612us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321443.552225:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6707 00000100:00100000:1.0:1767321443.552278:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985682048:0@lo:400:kworker.0 00000100:00100000:3.0:1767321443.552944:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6708 00000100:00100000:3.0:1767321443.552952:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985681920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321443.552970:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321443.553590:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985681920:12345-0@lo:38:kworker.0 Request processed in 635us (1943us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321443.553598:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6708 00000100:00100000:1.0:1767321443.553738:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321443.553752:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985682176:0@lo:400:kworker.0 00000100:00080000:2.0:1767321443.553795:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321443.553800:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321443.553805:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985681920:0@lo:38:kworker.0 00000100:00080000:2.0:1767321443.553818:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321443.553823:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321443.553826:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985681280:0@lo:38:kworker.0 00000100:00080000:2.0:1767321443.553840:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321443.553844:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321443.553848:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985681664:0@lo:38:kworker.0 00000100:00100000:3.0:1767321443.558477:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682048 00000100:00100000:3.0:1767321443.558493:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682048 00000100:00100000:3.0:1767321443.558811:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3756 00000100:00100000:3.0:1767321443.558821:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985682048:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321443.558996:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321443.559207:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985682048:12345-0@lo:400:kworker.0 Request processed in 386us (6890us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321443.559215:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3756 00000100:00100000:1.0:1767321443.563433:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985682048:0@lo:400:kworker.0 00010000:00010000:0.0:1767321444.190859:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321445.083509:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985682304:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321445.083605:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321445.084872:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682304 00000100:00100000:2.0:1767321445.084887:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682304 00000100:00100000:2.0:1767321445.085004:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2988 00000100:00100000:2.0:1767321445.085018:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985682304:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321445.085070:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321445.085256:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985682432:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321445.085310:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321445.085373:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682432 00000100:00100000:3.0:1767321445.085377:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682432 00000100:00100000:3.0:1767321445.085416:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2989 00000100:00100000:3.0:1767321445.085423:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985682432:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321445.085456:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321445.085494:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985682432:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (183us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321445.085502:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2989 00000100:00100000:3.0:1767321445.085559:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985682432:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321445.085695:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321445.085701:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321445.086289:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985682304:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1273us (2682us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321445.086299:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2988 00000100:00100000:0.0:1767321445.097338:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985682304:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321445.212475:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321446.235860:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321447.259537:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321447.595288:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321447.595402:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321447.595512:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674048 00000100:00100000:1.0:1767321447.595527:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674048 00000100:00100000:1.0:1767321447.595630:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920673920 00000100:00100000:1.0:1767321447.595635:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920673920 00000100:00080000:2.0:1767321447.595709:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321447.595960:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5827 00000100:00100000:1.0:1767321447.595976:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674048:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321447.596096:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674048:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 124us (827us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321447.596105:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5827 00000100:00100000:1.0:1767321447.596127:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6709 00000100:00100000:1.0:1767321447.596132:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920673920:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321447.596158:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321447.596236:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920673920:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 102us (833us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321447.596243:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6709 00010000:00010000:1.0:1767321448.286662:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321448.668207:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321448.668221:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321448.668224:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321448.668228:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321448.668233:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321448.668244:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6691 00000100:00080000:1.0:1767321448.668250:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321448.668259:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321448.668314:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1880 x1853163985682560/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321448.668346:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321448.668350:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321448.668352:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321448.668356:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6691 00000100:00080000:1.0:1767321448.668358:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321448.668362:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321448.668376:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff1c00 x1853163985682688/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321448.668420:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321448.668424:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321448.668426:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321448.668430:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6691 00000100:00080000:1.0:1767321448.668432:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321448.668436:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321448.668449:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0380 x1853163985682944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321448.668481:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321448.668485:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321448.668487:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321448.668490:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6691 00000100:00080000:1.0:1767321448.668493:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321448.668497:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321448.668508:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff0700 x1853163985683200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321448.668554:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985683072:0@lo:400:kworker.0 00000100:00100000:3.0:1767321448.668565:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985682816:0@lo:400:kworker.0 00000100:00100000:2.0:1767321448.668643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985682560:0@lo:38:kworker.0 00000100:00100000:0.0:1767321448.668695:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985683328:0@lo:400:kworker.0 00000100:00100000:1.0:1767321448.675183:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321448.675466:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683072 00000100:00100000:1.0:1767321448.675481:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683072 00000100:00100000:1.0:1767321448.675583:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5828 00000100:00100000:1.0:1767321448.675606:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985683072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321448.675681:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985683072:12345-0@lo:400:kworker.0 Request processed in 88us (523us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321448.675690:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5828 00000100:00100000:1.0:1767321448.675746:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985683072:0@lo:400:kworker.0 00000100:00080000:1.0:1767321448.675804:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321448.675831:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985683456:0@lo:400:kworker.0 00000100:00100000:1.0:1767321448.675868:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321448.675912:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683456 00000100:00100000:1.0:1767321448.675916:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683456 00000100:00100000:1.0:1767321448.675952:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3757 00000100:00100000:1.0:1767321448.675959:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985683456:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321448.675994:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985683456:12345-0@lo:400:kworker.0 Request processed in 36us (125us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321448.676000:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3757 00000100:00080000:1.0:1767321448.676014:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321448.676029:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985683456:0@lo:400:kworker.0 00000100:00100000:3.0:1767321448.678868:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321448.679131:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682816 00000100:00100000:1.0:1767321448.679142:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682816 00000100:00100000:1.0:1767321448.679256:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5829 00000100:00100000:1.0:1767321448.679267:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985682816:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321448.679350:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985682816:12345-0@lo:400:kworker.0 Request processed in 85us (493us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321448.679790:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5829 00000100:00080000:3.0:1767321448.679894:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321448.679946:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985682816:0@lo:400:kworker.0 00000100:00100000:2.0:1767321448.686988:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321448.687074:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682560 00000100:00100000:1.0:1767321448.687086:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682560 00000100:00100000:1.0:1767321448.687293:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6710 00000100:00100000:1.0:1767321448.687317:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985682560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321448.687351:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321448.687385:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985682688:0@lo:38:kworker.0 00000100:00100000:2.0:1767321448.687440:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321448.687443:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985682560:12345-0@lo:38:kworker.0 Request processed in 134us (464us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321448.687453:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6710 00000100:00100000:2.0:1767321448.687474:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985682944:0@lo:38:kworker.0 00000100:00100000:3.0:1767321448.687485:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682688 00000100:00100000:3.0:1767321448.687491:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682688 00000100:00100000:2.0:1767321448.687506:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321448.687520:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2168 00000100:00100000:2.0:1767321448.687526:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985683200:0@lo:38:kworker.0 00000100:00100000:3.0:1767321448.687526:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985682688:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321448.687538:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321448.687579:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985682688:12345-0@lo:38:kworker.0 Request processed in 52us (141us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321448.687585:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2168 00000100:00100000:3.0:1767321448.687628:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985682944 00000100:00100000:3.0:1767321448.687632:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985682944 00000100:00100000:3.0:1767321448.687654:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6711 00000100:00100000:3.0:1767321448.687659:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985682944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321448.687670:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321448.687697:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985682944:12345-0@lo:38:kworker.0 Request processed in 37us (191us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321448.687704:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6711 00000100:00100000:2.0:1767321448.690332:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321448.690465:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321448.690473:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321448.690478:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985682560:0@lo:38:kworker.0 00000100:00080000:2.0:1767321448.690494:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321448.690498:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321448.690500:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985682688:0@lo:38:kworker.0 00000100:00080000:2.0:1767321448.690513:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321448.690516:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321448.690519:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985682944:0@lo:38:kworker.0 00000100:00100000:0.0:1767321448.690984:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321448.691967:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683200 00000100:00100000:3.0:1767321448.691983:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683200 00000100:00100000:3.0:1767321448.692062:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6712 00000100:00100000:3.0:1767321448.692073:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985683200:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321448.692102:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321448.692233:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985683200:12345-0@lo:38:kworker.0 Request processed in 160us (4683us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321448.692241:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6712 00000100:00080000:2.0:1767321448.692792:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321448.692800:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321448.692805:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985683200:0@lo:38:kworker.0 00000100:00100000:1.0:1767321448.694358:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683328 00000100:00100000:1.0:1767321448.694374:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683328 00000100:00100000:1.0:1767321448.694488:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3758 00000100:00100000:1.0:1767321448.694500:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985683328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321448.694641:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985683328:12345-0@lo:400:kworker.0 Request processed in 144us (3657us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321448.694650:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3758 00000100:00100000:0.0:1767321448.694720:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985683328:0@lo:400:kworker.0 00000100:00080000:3.0:1767321448.694922:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321449.307944:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321450.203625:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321450.203891:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985683712:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321450.203995:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321450.204090:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683712 00000100:00100000:3.0:1767321450.204103:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683712 00000100:00100000:3.0:1767321450.204208:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2990 00000100:00100000:3.0:1767321450.204221:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985683712:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321450.204260:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767321450.204373:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321450.204873:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985683584:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321450.204925:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321450.205000:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683584 00000100:00100000:2.0:1767321450.205007:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683584 00000100:00100000:2.0:1767321450.205060:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2991 00000100:00100000:2.0:1767321450.205070:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985683584:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321450.205091:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321450.205133:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985683584:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (206us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321450.205140:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2991 00000100:00080000:1.0:1767321450.205328:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321450.205374:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985683584:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321450.205553:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985683712:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1335us (1557us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321450.205845:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985683712:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321450.205897:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2990 00010000:00010000:0.0:1767321450.331478:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321451.360057:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321452.380078:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321452.713883:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321452.713960:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674304 00000100:00100000:1.0:1767321452.713974:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674304 00000100:00100000:0.0:1767321452.713999:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321452.714096:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674176 00000100:00100000:3.0:1767321452.714102:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674176 00000100:00100000:1.0:1767321452.714139:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5830 00000100:00100000:1.0:1767321452.714151:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674304:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321452.714155:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6713 00000100:00100000:3.0:1767321452.714195:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920674176:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321452.714224:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321452.714298:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674304:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 149us (427us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321452.714313:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5830 00000100:00080000:2.0:1767321452.714398:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321452.714426:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920674176:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 262us (424us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321452.714433:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6713 00010000:00010000:1.0:1767321453.404553:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321453.787416:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321453.787754:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321453.787758:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321453.787763:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321453.787769:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321453.787780:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6696 00000100:00080000:0.0:1767321453.787786:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321453.787794:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321453.787846:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9180 x1853163985683840/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321453.787923:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985683840:0@lo:38:kworker.0 00000100:00100000:2.0:1767321453.788020:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321453.788108:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683840 00000100:00100000:3.0:1767321453.788120:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683840 00000100:00100000:3.0:1767321453.788203:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6714 00000100:00100000:3.0:1767321453.788214:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985683840:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321453.788242:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321453.788479:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321453.788483:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321453.788486:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321453.788490:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6696 00000100:00080000:0.0:1767321453.788493:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321453.788498:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321453.788527:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcda680 x1853163985683968/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321453.788866:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985683968:0@lo:38:kworker.0 00000100:00100000:2.0:1767321453.788908:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321453.789305:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985683840:12345-0@lo:38:kworker.0 Request processed in 1089us (1281us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321453.789318:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6714 00000100:00080000:2.0:1767321453.791783:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321453.791798:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321453.791804:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985683840:0@lo:38:kworker.0 00000100:00100000:2.0:1767321453.792120:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985684096:0@lo:400:kworker.0 00000100:00100000:2.0:1767321453.792205:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321453.792281:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684096 00000100:00100000:3.0:1767321453.792289:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684096 00000100:00100000:3.0:1767321453.792361:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5831 00000100:00100000:3.0:1767321453.792372:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985684096:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321453.792447:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321453.792453:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321453.792456:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321453.792462:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6696 00000100:00080000:0.0:1767321453.792466:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321453.792471:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321453.792549:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb100 x1853163985684224/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321453.792587:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321453.792590:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321453.792592:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321453.792596:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6696 00000100:00080000:0.0:1767321453.792598:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321453.792602:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321453.792609:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321453.792617:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985684352:0@lo:400:kworker.0 00000100:00100000:0.0:1767321453.792644:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321453.792671:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684352 00000100:00100000:0.0:1767321453.792674:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684352 00000100:00100000:0.0:1767321453.792699:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5832 00000100:00100000:0.0:1767321453.792704:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985684352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321453.792733:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985684352:12345-0@lo:400:kworker.0 Request processed in 28us (87us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321453.792741:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5832 00000100:00100000:0.0:1767321453.792773:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985684352:0@lo:400:kworker.0 00000100:00100000:0.0:1767321453.792796:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb480 x1853163985684480/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321453.792819:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985684608:0@lo:400:kworker.0 00000100:00100000:0.0:1767321453.792845:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321453.793058:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985683968 00000100:00100000:1.0:1767321453.793063:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985683968 00000100:00100000:1.0:1767321453.793079:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684608 00000100:00100000:1.0:1767321453.793081:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684608 00000100:00100000:1.0:1767321453.793128:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3759 00000100:00100000:1.0:1767321453.793135:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985684608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321453.793192:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985684608:12345-0@lo:400:kworker.0 Request processed in 57us (346us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321453.793198:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3759 00000100:00100000:1.0:1767321453.793225:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985684608:0@lo:400:kworker.0 00000100:00100000:1.0:1767321453.793263:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2169 00000100:00100000:1.0:1767321453.793268:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985683968:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321453.793290:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321453.793323:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985683968:12345-0@lo:38:kworker.0 Request processed in 53us (4414us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321453.793328:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2169 00000100:00100000:1.0:1767321453.793367:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985684736:0@lo:400:kworker.0 00000100:00100000:1.0:1767321453.793393:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321453.793419:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684736 00000100:00100000:1.0:1767321453.793422:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684736 00000100:00100000:1.0:1767321453.793449:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3760 00000100:00100000:1.0:1767321453.793455:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985684736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321453.793514:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985684736:12345-0@lo:400:kworker.0 Request processed in 46us (107us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321453.793521:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3760 00000100:00100000:1.0:1767321453.793545:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985684736:0@lo:400:kworker.0 00000100:00080000:1.0:1767321453.793578:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321453.793583:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321453.793585:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321453.793607:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985684224:0@lo:38:kworker.0 00000100:00100000:1.0:1767321453.793635:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321453.793650:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985684480:0@lo:38:kworker.0 00000100:00100000:1.0:1767321453.793669:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321453.793693:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321453.793699:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321453.793703:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985683968:0@lo:38:kworker.0 00000100:00100000:1.0:1767321453.793725:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684224 00000100:00100000:1.0:1767321453.793728:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684224 00000100:00100000:1.0:1767321453.793746:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684480 00000100:00100000:1.0:1767321453.793749:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684480 00000100:00100000:1.0:1767321453.793763:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6715 00000100:00100000:1.0:1767321453.793767:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985684224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321453.793778:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321453.793807:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985684224:12345-0@lo:38:kworker.0 Request processed in 38us (171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321453.793813:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6715 00000100:00100000:1.0:1767321453.793825:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6716 00000100:00100000:1.0:1767321453.793828:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985684480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321453.793835:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321453.793855:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985684480:12345-0@lo:38:kworker.0 Request processed in 25us (185us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321453.793860:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6716 00000100:00080000:1.0:1767321453.793883:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321453.793887:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321453.793890:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985684224:0@lo:38:kworker.0 00000100:00080000:1.0:1767321453.793896:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321453.793899:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321453.793901:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985684480:0@lo:38:kworker.0 00000100:00100000:3.0:1767321453.796412:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985684096:12345-0@lo:400:kworker.0 Request processed in 4025us (4192us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321453.796433:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5831 00000100:00100000:1.0:1767321453.796657:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985684096:0@lo:400:kworker.0 00010000:00010000:1.0:1767321454.430427:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321455.323726:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321455.323761:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985684864:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321455.323884:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321455.324108:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684864 00000100:00100000:2.0:1767321455.324122:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684864 00000100:00100000:2.0:1767321455.324249:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2992 00000100:00100000:2.0:1767321455.324264:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985684864:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321455.324312:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321455.324372:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985684864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (489us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321455.324382:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2992 00000100:00080000:0.0:1767321455.324942:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321455.325066:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985684864:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321455.335996:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321455.336044:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985684992:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321455.336219:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321455.336993:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985684992 00000100:00100000:3.0:1767321455.337008:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985684992 00000100:00100000:3.0:1767321455.337120:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2993 00000100:00100000:3.0:1767321455.337132:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985684992:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321455.337211:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321455.337314:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985684992:12345-0@lo:13:osp-pre-0-1.0 Request processed in 184us (1095us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321455.337324:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2993 00000100:00100000:1.0:1767321455.337631:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985684992:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321455.338030:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321455.452811:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321456.475532:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321457.499404:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321457.836207:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321457.836290:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321457.836362:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674560 00000100:00100000:3.0:1767321457.836371:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674560 00000100:00100000:3.0:1767321457.836467:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674432 00000100:00100000:3.0:1767321457.836470:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674432 00000100:00100000:3.0:1767321457.836489:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5833 00000100:00100000:3.0:1767321457.836498:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674560:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321457.836574:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674560:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 77us (375us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321457.836583:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5833 00000100:00100000:3.0:1767321457.836601:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6717 00000100:00100000:3.0:1767321457.836605:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920674432:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321457.836621:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321457.836641:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920674432:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 35us (352us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321457.836645:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6717 00000100:00080000:0.0:1767321457.837022:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321458.527109:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321458.907455:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321458.907466:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321458.907469:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321458.907473:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321458.907477:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321458.907485:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6702 00000100:00080000:3.0:1767321458.907489:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321458.907496:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321458.907533:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff6300 x1853163985685120/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321458.907562:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321458.907566:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321458.907568:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321458.907572:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6702 00000100:00080000:3.0:1767321458.907574:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321458.907578:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321458.907590:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3ff5f80 x1853163985685248/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321458.907639:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321458.907643:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321458.907646:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321458.907649:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6702 00000100:00080000:3.0:1767321458.907652:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321458.907655:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321458.907666:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73c380 x1853163985685504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321458.907689:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321458.907692:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321458.907704:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321458.907707:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6702 00000100:00080000:3.0:1767321458.907710:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321458.907713:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321458.907732:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1f80 x1853163985685760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321458.907771:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985685632:0@lo:400:kworker.0 00000100:00100000:3.0:1767321458.907831:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321458.907975:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985686016:0@lo:400:kworker.0 00000100:00100000:0.0:1767321458.908009:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321458.908065:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321458.908071:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985685376:0@lo:400:kworker.0 00000100:00100000:2.0:1767321458.908248:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985685120:0@lo:38:kworker.0 00000100:00100000:3.0:1767321458.908365:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321458.908371:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985685888:0@lo:400:kworker.0 00000100:00100000:3.0:1767321458.908394:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321458.908629:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321458.908661:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985685248:0@lo:38:kworker.0 00000100:00100000:2.0:1767321458.908687:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321458.908706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985685504:0@lo:38:kworker.0 00000100:00100000:2.0:1767321458.908737:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321458.908754:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985685760:0@lo:38:kworker.0 00000100:00100000:2.0:1767321458.908772:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321458.908803:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985685632 00000100:00100000:2.0:1767321458.908811:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985685632 00000100:00100000:1.0:1767321458.908859:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686016 00000100:00100000:1.0:1767321458.908864:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686016 00000100:00100000:2.0:1767321458.908867:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985685120 00000100:00100000:2.0:1767321458.908869:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985685120 00000100:00100000:2.0:1767321458.908880:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985685504 00000100:00100000:1.0:1767321458.908893:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985685888 00000100:00100000:2.0:1767321458.908893:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985685504 00000100:00100000:1.0:1767321458.908895:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985685888 00000100:00100000:2.0:1767321458.908901:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985685760 00000100:00100000:2.0:1767321458.908903:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985685760 00000100:00100000:2.0:1767321458.908914:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5834 00000100:00100000:2.0:1767321458.908922:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985685632:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321458.908923:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3761 00000100:00100000:1.0:1767321458.908928:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985686016:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321458.908968:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985686016:12345-0@lo:400:kworker.0 Request processed in 39us (957us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321458.908968:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985685632:12345-0@lo:400:kworker.0 Request processed in 46us (1135us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321458.908976:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3761 00000100:00100000:2.0:1767321458.908976:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5834 00000100:00100000:1.0:1767321458.908990:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3762 00000100:00100000:2.0:1767321458.908991:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6718 00000100:00100000:2.0:1767321458.908994:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985685120:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321458.908996:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985685888:12345-0@lo:400:kworker.0 00010000:00020000:2.0:1767321458.909011:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321458.909017:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985685888:12345-0@lo:400:kworker.0 Request processed in 22us (623us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321458.909023:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3762 00000100:00100000:2.0:1767321458.909035:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985685120:12345-0@lo:38:kworker.0 Request processed in 40us (406us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321458.909041:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6718 00000100:00100000:2.0:1767321458.909051:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6719 00000100:00100000:2.0:1767321458.909055:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985685504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321458.909064:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321458.909082:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985685504:12345-0@lo:38:kworker.0 Request processed in 27us (345us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321458.909087:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6719 00000100:00100000:2.0:1767321458.909096:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6720 00000100:00100000:2.0:1767321458.909099:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985685760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321458.909106:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321458.909123:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a593aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985685760:12345-0@lo:38:kworker.0 Request processed in 23us (350us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321458.909128:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6720 00000100:00080000:2.0:1767321458.909160:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321458.909184:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321458.909188:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985685120:0@lo:38:kworker.0 00000100:00100000:1.0:1767321458.909189:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985685248 00000100:00100000:1.0:1767321458.909193:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985685248 00000100:00080000:2.0:1767321458.909199:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321458.909202:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321458.909205:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985685504:0@lo:38:kworker.0 00000100:00080000:2.0:1767321458.909212:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321458.909215:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321458.909217:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985685760:0@lo:38:kworker.0 00000100:00100000:1.0:1767321458.909222:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2170 00000100:00100000:1.0:1767321458.909230:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985685248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321458.909240:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321458.909249:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985685888:0@lo:400:kworker.0 00000100:00100000:2.0:1767321458.909258:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985685632:0@lo:400:kworker.0 00000100:00100000:1.0:1767321458.909275:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985685248:12345-0@lo:38:kworker.0 Request processed in 48us (587us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767321458.909276:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321458.909280:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321458.909282:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985685248:0@lo:38:kworker.0 00000100:00100000:1.0:1767321458.909294:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2170 00000100:00080000:0.0:1767321458.909498:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321458.909503:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321458.909505:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321458.909520:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321458.909554:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985686016:0@lo:400:kworker.0 00000100:00100000:3.0:1767321458.909614:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985685376 00000100:00100000:3.0:1767321458.909618:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985685376 00000100:00100000:3.0:1767321458.909656:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5835 00000100:00100000:3.0:1767321458.909662:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985685376:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321458.909670:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321458.909694:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985685376:12345-0@lo:400:kworker.0 Request processed in 32us (173us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321458.909699:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5835 00000100:00100000:0.0:1767321458.909710:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985685376:0@lo:400:kworker.0 00010000:00010000:1.0:1767321459.547362:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321460.443481:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985686272:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321460.443551:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321460.443634:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686272 00000100:00100000:1.0:1767321460.443643:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686272 00000100:00100000:0.0:1767321460.443650:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985686144:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321460.443689:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321460.445212:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321460.445502:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686144 00000100:00100000:1.0:1767321460.445512:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686144 00000100:00100000:1.0:1767321460.445565:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2994 00000100:00100000:1.0:1767321460.445575:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985686272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321460.445605:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321460.445671:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985686272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (2119us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321460.445678:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2994 00000100:00100000:3.0:1767321460.445701:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2995 00000100:00100000:3.0:1767321460.445718:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985686144:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321460.445722:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:3.0:1767321460.445738:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321460.445767:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985686272:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321460.445775:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985686144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (2084us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321460.445783:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2995 00000100:00100000:0.0:1767321460.445890:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985686144:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321460.571335:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321461.595491:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321462.619512:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321462.954014:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321462.954901:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674816 00000100:00100000:1.0:1767321462.954915:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674816 00000100:00100000:0.0:1767321462.955001:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321462.955006:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5836 00000100:00100000:1.0:1767321462.955018:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674816:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321462.955096:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920674816:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 80us (1093us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321462.955104:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5836 00000100:00100000:1.0:1767321462.955120:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674688 00000100:00100000:1.0:1767321462.955123:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674688 00000100:00100000:1.0:1767321462.955138:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6721 00000100:00100000:1.0:1767321462.955142:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920674688:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321462.955161:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321462.955211:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920674688:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 67us (214us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321462.955218:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6721 00000100:00080000:3.0:1767321462.955801:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321463.644812:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321464.027266:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321464.027274:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321464.027277:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321464.027281:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321464.027285:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321464.027293:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6707 00000100:00080000:3.0:1767321464.027307:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321464.027323:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321464.027362:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3100 x1853163985686400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321464.027386:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321464.027389:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321464.027391:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321464.027394:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6707 00000100:00080000:3.0:1767321464.027396:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321464.027400:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321464.027410:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1c00 x1853163985686528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321464.027486:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985686656:0@lo:400:kworker.0 00000100:00100000:1.0:1767321464.027553:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321464.027668:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985686400:0@lo:38:kworker.0 00000100:00100000:0.0:1767321464.027695:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686656 00000100:00100000:2.0:1767321464.027702:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321464.027706:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686656 00000100:00100000:2.0:1767321464.027733:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985686528:0@lo:38:kworker.0 00000100:00100000:2.0:1767321464.027758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321464.027766:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686400 00000100:00100000:0.0:1767321464.027770:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686400 00000100:00100000:0.0:1767321464.027796:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5837 00000100:00100000:0.0:1767321464.027806:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985686656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321464.027807:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6722 00000100:00100000:2.0:1767321464.027817:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985686400:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321464.027837:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321464.027876:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985686400:12345-0@lo:38:kworker.0 Request processed in 58us (173us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767321464.027876:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321464.027884:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321464.027888:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767321464.027892:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6722 00000100:00080000:3.0:1767321464.027895:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6707 00000100:00080000:3.0:1767321464.027898:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321464.027902:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321464.027917:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d1180 x1853163985686784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321464.027935:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321464.027941:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00080000:3.0:1767321464.027943:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767321464.027949:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985686400:0@lo:38:kworker.0 00000100:00080000:3.0:1767321464.027949:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321464.027952:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321464.027959:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6707 00000100:00080000:3.0:1767321464.027962:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321464.027966:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321464.027978:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b34d3100 x1853163985687040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321464.027986:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985686784:0@lo:38:kworker.0 00000100:00100000:2.0:1767321464.028014:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321464.028034:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985687040:0@lo:38:kworker.0 00000100:00100000:2.0:1767321464.028052:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321464.028073:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686784 00000100:00100000:2.0:1767321464.028077:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686784 00000100:00100000:2.0:1767321464.028093:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985687040 00000100:00100000:2.0:1767321464.028095:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985687040 00000100:00100000:2.0:1767321464.028108:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6723 00000100:00100000:2.0:1767321464.028112:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985686784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321464.028122:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321464.028149:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985686784:12345-0@lo:38:kworker.0 Request processed in 35us (134us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321464.028155:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6723 00000100:00100000:2.0:1767321464.028184:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6724 00000100:00100000:2.0:1767321464.028188:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985687040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321464.028197:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321464.028219:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985687040:12345-0@lo:38:kworker.0 Request processed in 29us (166us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321464.028225:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6724 00000100:00080000:2.0:1767321464.028255:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321464.028259:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321464.028263:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985687040:0@lo:38:kworker.0 00000100:00080000:2.0:1767321464.028270:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321464.028273:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321464.028275:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985686784:0@lo:38:kworker.0 00000100:00100000:2.0:1767321464.028320:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985687296:0@lo:400:kworker.0 00000100:00100000:2.0:1767321464.028346:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321464.028364:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985686656:12345-0@lo:400:kworker.0 Request processed in 559us (808us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321464.028372:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5837 00000100:00080000:2.0:1767321464.028383:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321464.028396:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985687168:0@lo:400:kworker.0 00000100:00100000:2.0:1767321464.028399:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985687296 00000100:00100000:2.0:1767321464.028404:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985687296 00000100:00100000:0.0:1767321464.028426:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321464.028480:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985686912:0@lo:400:kworker.0 00000100:00100000:3.0:1767321464.028513:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321464.028537:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686912 00000100:00100000:3.0:1767321464.028541:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686912 00000100:00100000:3.0:1767321464.028562:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5838 00000100:00100000:3.0:1767321464.028567:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985686912:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321464.028596:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985686912:12345-0@lo:400:kworker.0 Request processed in 28us (83us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321464.028602:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5838 00000100:00100000:3.0:1767321464.028628:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985686912:0@lo:400:kworker.0 00000100:00080000:3.0:1767321464.028650:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321464.028653:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321464.028697:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985686656:0@lo:400:kworker.0 00000100:00100000:1.0:1767321464.028720:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985686528 00000100:00100000:1.0:1767321464.028723:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985686528 00000100:00100000:1.0:1767321464.028744:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2171 00000100:00100000:1.0:1767321464.028748:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985686528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321464.028758:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321464.028790:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985686528:12345-0@lo:38:kworker.0 Request processed in 40us (1031us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321464.028796:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2171 00000100:00080000:1.0:1767321464.028834:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321464.028838:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321464.028841:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985686528:0@lo:38:kworker.0 00000100:00100000:2.0:1767321464.032729:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985687168 00000100:00100000:2.0:1767321464.032746:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985687168 00000100:00100000:3.0:1767321464.032815:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3763 00000100:00080000:0.0:1767321464.032874:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321464.032914:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985687296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321464.033055:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985687296:12345-0@lo:400:kworker.0 Request processed in 229us (4707us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321464.033144:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985687296:0@lo:400:kworker.0 00000100:00100000:2.0:1767321464.033535:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3764 00000100:00100000:2.0:1767321464.033546:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985687168:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321464.033599:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985687168:12345-0@lo:400:kworker.0 Request processed in 56us (5173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321464.033606:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3764 00000100:00100000:0.0:1767321464.033664:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985687168:0@lo:400:kworker.0 00000100:00100000:3.0:1767321464.033776:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3763 00010000:00010000:2.0:1767321464.669143:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321465.564198:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985687424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321465.564273:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321465.564382:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985687424 00000100:00100000:3.0:1767321465.564390:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985687424 00000100:00100000:3.0:1767321465.564448:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2996 00000100:00100000:3.0:1767321465.564457:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985687424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321465.564488:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321465.564550:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985687424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (277us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321465.564558:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2996 00000100:00100000:3.0:1767321465.564607:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985687424:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321465.565818:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321465.565835:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985687552:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321465.565890:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321465.566069:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985687552 00000100:00100000:1.0:1767321465.566076:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985687552 00000100:00100000:1.0:1767321465.566123:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2997 00000100:00100000:1.0:1767321465.566132:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985687552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321465.566157:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321465.566225:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985687552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (334us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321465.566234:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2997 00000100:00100000:0.0:1767321465.566362:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985687552:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321465.692368:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321466.715587:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321467.739788:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321468.074848:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321468.075084:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920675072 00000100:00100000:1.0:1767321468.075101:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920675072 00000100:00100000:1.0:1767321468.075255:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5839 00000100:00100000:1.0:1767321468.075267:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920675072:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321468.075437:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920675072:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 171us (604us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321468.075446:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5839 00000100:00080000:0.0:1767321468.075688:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321468.088590:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:2.0:1767321468.088634:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920675200 offset 0 length 520: 4 00000100:00100000:3.0:1767321468.088868:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920674944 00000100:00100000:3.0:1767321468.088878:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920674944 00000100:00100000:3.0:1767321468.089342:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6725 00000100:00100000:3.0:1767321468.089351:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920674944:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321468.089377:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321468.089434:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920674944:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 83us (854us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321468.089441:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6725 00000100:00100000:3.0:1767321468.123312:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321443/real 1767321443] req@ffff8ce19dcdad80 x1853163985681152/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321468 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321468.123347:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce19dcdad80 x1853163985681152/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321468 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321468.123366:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321468.123387:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321468.123391:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985681152:0@lo:250:kworker.0 00000100:00080000:0.0:1767321468.123807:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321468.123815:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321468.123819:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321468.123824:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6691 00000100:00080000:0.0:1767321468.123828:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321468.123835:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321468.123868:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22680 x1853163985687680/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321468.123901:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321468.123905:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321468.123907:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321468.123911:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6712 00000100:00080000:0.0:1767321468.123913:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321468.123916:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321468.123931:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21f80 x1853163985687808/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321468.123940:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321468.123943:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321468.123944:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321468.123947:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6712 00000100:00080000:0.0:1767321468.123950:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321468.123953:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321468.123963:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22d80 x1853163985687936/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321468.123990:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321468.123994:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321468.123997:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321468.124000:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6712 00000100:00080000:0.0:1767321468.124002:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321468.124005:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321468.124016:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe20000 x1853163985688192/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321468.124036:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321468.124039:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321468.124041:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321468.124044:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6712 00000100:00080000:0.0:1767321468.124047:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321468.124049:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321468.124059:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74380 x1853163985688448/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321468.124337:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985688704:0@lo:400:kworker.0 00000100:00100000:3.0:1767321468.124402:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321468.124472:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321468.124479:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985688064:0@lo:400:kworker.0 00000100:00100000:3.0:1767321468.124503:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321468.124528:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688064 00000100:00100000:1.0:1767321468.124547:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688064 00000100:00100000:3.0:1767321468.124556:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985688576:0@lo:400:kworker.0 00000100:00100000:3.0:1767321468.124583:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321468.124605:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321468.124610:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5840 00000100:00100000:3.0:1767321468.124613:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985688320:0@lo:400:kworker.0 00000100:00100000:1.0:1767321468.124623:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985688064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321468.124637:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321468.124681:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688320 00000100:00100000:3.0:1767321468.124686:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688320 00000100:00100000:1.0:1767321468.124691:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985688064:12345-0@lo:400:kworker.0 Request processed in 68us (184us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321468.124700:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5840 00000100:00100000:3.0:1767321468.124718:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5841 00000100:00100000:3.0:1767321468.124728:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985688320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321468.124756:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985688064:0@lo:400:kworker.0 00000100:00100000:3.0:1767321468.124767:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985688320:12345-0@lo:400:kworker.0 Request processed in 41us (129us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321468.124775:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5841 00000100:00100000:1.0:1767321468.124808:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985687680:0@lo:250:kworker.0 00000100:00100000:3.0:1767321468.124808:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985688320:0@lo:400:kworker.0 00000400:00000100:1.0:1767321468.124842:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985687680 offset 0 length 520: 4 00000100:00100000:1.0:1767321468.124863:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985687808:0@lo:38:kworker.0 00000100:00100000:1.0:1767321468.124886:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321468.125089:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985687936:0@lo:38:kworker.0 00000100:00100000:1.0:1767321468.125116:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321468.125135:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985688192:0@lo:38:kworker.0 00000100:00100000:3.0:1767321468.125150:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985687808 00000100:00100000:1.0:1767321468.125155:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321468.125157:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985687808 00000100:00100000:3.0:1767321468.125200:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688192 00000100:00100000:3.0:1767321468.125203:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688192 00000100:00100000:3.0:1767321468.125218:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6726 00000100:00100000:3.0:1767321468.125223:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985687808:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321468.125224:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985688448:0@lo:38:kworker.0 00010000:00020000:3.0:1767321468.125256:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321468.125264:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321468.125300:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767321468.125300:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985687808:12345-0@lo:38:kworker.0 Request processed in 74us (412us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767321468.125309:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321468.125313:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6726 00000100:00100000:1.0:1767321468.125317:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985687808:0@lo:38:kworker.0 00000100:00100000:3.0:1767321468.125330:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688448 00000100:00100000:3.0:1767321468.125335:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688448 00000100:00100000:1.0:1767321468.125348:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985687936 00000100:00100000:3.0:1767321468.125353:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6727 00000100:00100000:1.0:1767321468.125355:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985687936 00000100:00100000:3.0:1767321468.125359:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985688192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321468.125370:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321468.125383:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2172 00000100:00100000:1.0:1767321468.125388:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985687936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321468.125401:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321468.125406:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985688192:12345-0@lo:38:kworker.0 Request processed in 45us (250us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321468.125427:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6727 00000100:00100000:1.0:1767321468.125433:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985687936:12345-0@lo:38:kworker.0 Request processed in 41us (315us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321468.125442:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6728 00000100:00100000:1.0:1767321468.125443:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2172 00000100:00100000:3.0:1767321468.125451:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985688448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321468.125461:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767321468.125481:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321468.125487:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321468.125492:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985688448:12345-0@lo:38:kworker.0 Request processed in 39us (226us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321468.125495:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985687936:0@lo:38:kworker.0 00000100:00100000:3.0:1767321468.125503:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6728 00000100:00080000:1.0:1767321468.125509:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321468.125514:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321468.125517:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985688192:0@lo:38:kworker.0 00000100:00080000:1.0:1767321468.125525:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321468.125528:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321468.125530:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985688448:0@lo:38:kworker.0 00000100:00100000:0.0:1767321468.125727:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688704 00000100:00100000:0.0:1767321468.125732:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688704 00000100:00100000:0.0:1767321468.125774:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688576 00000100:00100000:0.0:1767321468.125778:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688576 00000100:00100000:0.0:1767321468.125812:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3765 00000100:00100000:0.0:1767321468.125819:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985688704:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321468.125864:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985688704:12345-0@lo:400:kworker.0 Request processed in 46us (1461us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321468.125870:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3765 00000100:00100000:0.0:1767321468.125883:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3766 00000100:00100000:0.0:1767321468.125890:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985688576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321468.125931:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985688576:12345-0@lo:400:kworker.0 Request processed in 42us (1347us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321468.125938:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3766 00000100:00080000:0.0:1767321468.125957:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321468.125960:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321468.125962:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321468.125983:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985688704:0@lo:400:kworker.0 00000100:00100000:3.0:1767321468.126121:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985688576:0@lo:400:kworker.0 00010000:00010000:0.0:1767321468.764621:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321469.787457:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321470.685927:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985688832:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321470.685999:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321470.686030:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688832 00000100:00100000:2.0:1767321470.686039:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688832 00000100:00100000:2.0:1767321470.686104:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2998 00000100:00080000:0.0:1767321470.686117:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321470.686123:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985688832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321470.686158:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321470.686246:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985688832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 133us (247us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321470.686253:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2998 00000100:00100000:0.0:1767321470.686354:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985688832:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321470.686591:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321470.686602:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985688960:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321470.686632:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321470.686715:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985688960 00000100:00100000:1.0:1767321470.686720:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985688960 00000100:00100000:1.0:1767321470.686765:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2999 00000100:00100000:1.0:1767321470.686773:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985688960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321470.686790:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321470.686831:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985688960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (198us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321470.686837:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2999 00000100:00080000:0.0:1767321470.686925:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321470.686977:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985688960:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321470.811514:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321471.838471:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321472.859761:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321473.200004:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321473.200046:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321473.200228:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920675456 00000100:00100000:2.0:1767321473.200237:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920675456 00000100:00100000:2.0:1767321473.200301:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920675328 00000100:00100000:2.0:1767321473.200305:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920675328 00000100:00100000:2.0:1767321473.200326:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5842 00000100:00100000:2.0:1767321473.200339:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920675456:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767321473.200352:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321473.200386:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6729 00000100:00100000:3.0:1767321473.200394:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920675328:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321473.200415:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321473.200429:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920675456:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (435us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321473.200439:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5842 00000100:00100000:3.0:1767321473.200456:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920675328:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 60us (407us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321473.200464:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6729 00000100:00080000:0.0:1767321473.243299:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321473.243311:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321473.243315:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321473.243320:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321473.243332:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321473.243341:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6716 00000100:00080000:0.0:1767321473.243346:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321473.243354:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321473.243400:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74a80 x1853163985689088/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321473.243426:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321473.243429:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321473.243431:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321473.243434:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6716 00000100:00080000:0.0:1767321473.243437:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321473.243442:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321473.243454:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163985689216/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321473.243490:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321473.243494:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321473.243496:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321473.243500:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6716 00000100:00080000:0.0:1767321473.243502:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767321473.243505:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985689088:0@lo:38:kworker.0 00000100:00080000:0.0:1767321473.243509:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321473.243525:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74e00 x1853163985689472/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321473.243562:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321473.243566:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321473.243567:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321473.243571:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6716 00000100:00080000:0.0:1767321473.243574:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321473.243578:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321473.243583:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321473.243596:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db75180 x1853163985689728/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321473.243647:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985689216:0@lo:38:kworker.0 00000100:00100000:0.0:1767321473.243667:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985689984:0@lo:400:kworker.0 00000100:00100000:1.0:1767321473.243675:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321473.243709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985689472:0@lo:38:kworker.0 00000100:00100000:0.0:1767321473.243712:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321473.243741:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321473.243747:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321473.243755:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985689344:0@lo:400:kworker.0 00000100:00100000:0.0:1767321473.243954:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321473.244018:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985689856:0@lo:400:kworker.0 00000100:00100000:0.0:1767321473.244045:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321473.244061:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321473.244066:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985689600:0@lo:400:kworker.0 00000100:00100000:0.0:1767321473.244084:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321473.244279:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689216 00000100:00100000:0.0:1767321473.244286:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689216 00000100:00100000:0.0:1767321473.244318:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2173 00000100:00100000:0.0:1767321473.244325:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985689216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321473.244344:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321473.244379:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985689216:12345-0@lo:38:kworker.0 Request processed in 53us (703us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321473.244386:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2173 00000100:00100000:0.0:1767321473.244407:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689088 00000100:00100000:0.0:1767321473.244411:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689088 00000100:00100000:0.0:1767321473.244426:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689472 00000100:00100000:0.0:1767321473.244428:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689472 00000100:00100000:0.0:1767321473.244441:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689344 00000100:00100000:0.0:1767321473.244443:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689344 00000100:00100000:2.0:1767321473.244458:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689984 00000100:00100000:2.0:1767321473.244464:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689984 00000100:00100000:0.0:1767321473.244471:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689600 00000100:00100000:0.0:1767321473.244476:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689600 00000100:00100000:0.0:1767321473.244499:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5843 00000100:00100000:0.0:1767321473.244505:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985689344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321473.244506:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689856 00000100:00100000:2.0:1767321473.244512:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689856 00000100:00100000:2.0:1767321473.244539:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3767 00000100:00100000:2.0:1767321473.244547:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985689984:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321473.244599:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985689984:12345-0@lo:400:kworker.0 Request processed in 53us (891us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321473.244605:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3767 00000100:00100000:2.0:1767321473.244617:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3768 00000100:00100000:2.0:1767321473.244622:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985689856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321473.244650:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985689856:12345-0@lo:400:kworker.0 Request processed in 28us (604us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321473.244655:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3768 00000100:00100000:2.0:1767321473.244802:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985689984:0@lo:400:kworker.0 00000100:00100000:2.0:1767321473.244835:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5844 00000100:00100000:2.0:1767321473.244842:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985689600:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321473.244872:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985689600:12345-0@lo:400:kworker.0 Request processed in 31us (786us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321473.244877:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5844 00000100:00100000:2.0:1767321473.244891:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6730 00000100:00100000:2.0:1767321473.244894:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985689088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321473.244905:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321473.244928:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985689088:12345-0@lo:38:kworker.0 Request processed in 32us (1349us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321473.244934:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6730 00000100:00100000:2.0:1767321473.244946:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6731 00000100:00100000:2.0:1767321473.244949:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985689472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321473.244957:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321473.244978:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985689472:12345-0@lo:38:kworker.0 Request processed in 27us (1238us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321473.244983:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6731 00000100:00100000:2.0:1767321473.245007:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985689600:0@lo:400:kworker.0 00000100:00100000:2.0:1767321473.245017:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985689856:0@lo:400:kworker.0 00000100:00100000:0.0:1767321473.245122:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985689344:12345-0@lo:400:kworker.0 Request processed in 617us (1166us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321473.245130:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5843 00000100:00100000:2.0:1767321473.246597:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985689344:0@lo:400:kworker.0 00000100:00080000:0.0:1767321473.251325:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321473.251412:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321473.251417:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321473.255191:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321473.255206:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321473.255212:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985689472:0@lo:38:kworker.0 00000100:00100000:1.0:1767321473.255226:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985689728:0@lo:38:kworker.0 00000100:00100000:1.0:1767321473.255292:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321473.255330:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321473.255335:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321473.255340:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985689088:0@lo:38:kworker.0 00000100:00080000:1.0:1767321473.255353:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321473.255357:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321473.255360:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985689216:0@lo:38:kworker.0 00000100:00100000:3.0:1767321473.255458:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985689728 00000100:00100000:3.0:1767321473.255465:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985689728 00000100:00100000:3.0:1767321473.255506:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6732 00000100:00100000:3.0:1767321473.255526:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985689728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321473.255549:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321473.255603:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985689728:12345-0@lo:38:kworker.0 Request processed in 89us (311us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321473.255613:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6732 00000100:00080000:1.0:1767321473.255679:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321473.255684:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321473.255687:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985689728:0@lo:38:kworker.0 00010000:00010000:2.0:1767321473.883419:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321474.907452:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321475.803665:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321475.803674:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321475.803690:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985690240:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321475.803691:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985690112:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321475.805338:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321475.805450:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985690112 00000100:00100000:3.0:1767321475.805460:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985690112 00000100:00080000:0.0:1767321475.805517:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321475.805572:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3000 00000100:00100000:3.0:1767321475.805582:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985690112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321475.805614:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321475.805666:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985690112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321475.805673:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3000 00000100:00100000:2.0:1767321475.805713:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321475.805781:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985690112:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321475.806300:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985690240 00000100:00100000:1.0:1767321475.806306:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985690240 00000100:00080000:2.0:1767321475.806423:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321475.806478:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3001 00000100:00100000:1.0:1767321475.806486:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985690240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321475.806505:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321475.806547:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985690240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (833us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321475.806554:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3001 00000100:00100000:1.0:1767321475.806590:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985690240:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321475.938578:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321476.955409:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321477.980260:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321478.314886:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321478.314989:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920675712 00000100:00100000:0.0:1767321478.315000:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920675712 00000100:00100000:0.0:1767321478.315099:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5845 00000100:00100000:0.0:1767321478.315110:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920675712:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321478.315233:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920675712:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 125us (353us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321478.315242:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5845 00000100:00080000:2.0:1767321478.316529:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321478.321707:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321478.322088:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920675584 00000100:00100000:1.0:1767321478.322099:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920675584 00000100:00100000:1.0:1767321478.322257:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6733 00000100:00100000:1.0:1767321478.322266:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920675584:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321478.322289:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321478.322414:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920675584:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 146us (712us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321478.322422:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6733 00000100:00080000:0.0:1767321478.363489:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321478.363501:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321478.363504:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321478.363507:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321478.363511:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321478.363519:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6721 00000100:00080000:0.0:1767321478.363524:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321478.363529:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321478.363569:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db77480 x1853163985690368/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321478.363603:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321478.363607:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321478.363609:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321478.363612:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6721 00000100:00080000:0.0:1767321478.363615:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321478.363619:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321478.363636:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74700 x1853163985690496/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321478.363679:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321478.363683:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321478.363685:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321478.363689:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6721 00000100:00080000:0.0:1767321478.363691:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321478.363695:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321478.363707:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76300 x1853163985690752/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321478.363738:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321478.363742:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321478.363743:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321478.363747:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6721 00000100:00080000:0.0:1767321478.363750:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321478.363754:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321478.363764:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321478.363766:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74e00 x1853163985691008/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321478.363786:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985690624:0@lo:400:kworker.0 00000100:00100000:0.0:1767321478.363811:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985691136:0@lo:400:kworker.0 00000100:00100000:0.0:1767321478.363857:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321478.363857:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321478.363934:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321478.363944:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985691264:0@lo:400:kworker.0 00000100:00100000:2.0:1767321478.363972:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321478.364005:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691136 00000100:00100000:2.0:1767321478.364014:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691136 00000100:00100000:2.0:1767321478.364092:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691264 00000100:00100000:2.0:1767321478.364096:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691264 00000100:00100000:2.0:1767321478.364132:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3769 00000100:00100000:2.0:1767321478.364142:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985691136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321478.364206:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985691136:12345-0@lo:400:kworker.0 Request processed in 65us (353us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321478.364213:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3769 00000100:00100000:2.0:1767321478.364231:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3770 00000100:00100000:2.0:1767321478.364238:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985691264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321478.364269:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985691264:12345-0@lo:400:kworker.0 Request processed in 32us (296us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321478.364274:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3770 00000100:00080000:2.0:1767321478.364299:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321478.364303:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321478.364329:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985690880:0@lo:400:kworker.0 00000100:00100000:2.0:1767321478.364368:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321478.364396:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985690624 00000100:00100000:2.0:1767321478.364400:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985690624 00000100:00100000:2.0:1767321478.364430:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985690880 00000100:00100000:2.0:1767321478.364433:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985690880 00000100:00100000:2.0:1767321478.364450:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5846 00000100:00100000:2.0:1767321478.364455:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985690624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321478.364487:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985690624:12345-0@lo:400:kworker.0 Request processed in 31us (635us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321478.364492:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5846 00000100:00100000:2.0:1767321478.364504:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5847 00000100:00100000:2.0:1767321478.364508:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985690880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321478.364537:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985690880:12345-0@lo:400:kworker.0 Request processed in 29us (169us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321478.364542:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5847 00000100:00080000:2.0:1767321478.364581:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321478.364773:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985690368:0@lo:38:kworker.0 00000100:00100000:1.0:1767321478.364804:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321478.365066:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985691136:0@lo:400:kworker.0 00000100:00100000:0.0:1767321478.365105:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985691264:0@lo:400:kworker.0 00000100:00100000:0.0:1767321478.365136:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985690880:0@lo:400:kworker.0 00000100:00100000:3.0:1767321478.365277:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985690368 00000100:00100000:3.0:1767321478.365284:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985690368 00000100:00100000:3.0:1767321478.365307:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6734 00000100:00100000:3.0:1767321478.365314:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985690368:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321478.365333:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321478.365382:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985690368:12345-0@lo:38:kworker.0 Request processed in 67us (577us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321478.365388:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6734 00000100:00100000:3.0:1767321478.365417:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985690624:0@lo:400:kworker.0 00000100:00080000:0.0:1767321478.365592:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321478.365597:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321478.365600:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985690368:0@lo:38:kworker.0 00000100:00100000:0.0:1767321478.365607:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985690496:0@lo:38:kworker.0 00000100:00100000:0.0:1767321478.365635:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321478.365669:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985690752:0@lo:38:kworker.0 00000100:00100000:0.0:1767321478.365688:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321478.365702:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985691008:0@lo:38:kworker.0 00000100:00100000:0.0:1767321478.365718:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321478.365746:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985690752 00000100:00100000:0.0:1767321478.365750:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985690752 00000100:00100000:0.0:1767321478.365768:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691008 00000100:00100000:0.0:1767321478.365771:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691008 00000100:00100000:0.0:1767321478.365785:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6735 00000100:00100000:0.0:1767321478.365790:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985690752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321478.365800:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321478.365826:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985690752:12345-0@lo:38:kworker.0 Request processed in 35us (136us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321478.365831:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6735 00000100:00100000:0.0:1767321478.365842:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6736 00000100:00100000:0.0:1767321478.365847:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985691008:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321478.365855:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321478.365888:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985691008:12345-0@lo:38:kworker.0 Request processed in 41us (169us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321478.365894:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6736 00000100:00080000:0.0:1767321478.365917:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321478.365921:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321478.365924:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985690752:0@lo:38:kworker.0 00000100:00080000:0.0:1767321478.365931:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321478.365934:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321478.365936:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985691008:0@lo:38:kworker.0 00000100:00100000:3.0:1767321478.366263:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985690496 00000100:00100000:3.0:1767321478.366268:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985690496 00000100:00100000:3.0:1767321478.366297:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2174 00000100:00100000:3.0:1767321478.366306:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985690496:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321478.366320:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321478.366374:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985690496:12345-0@lo:38:kworker.0 Request processed in 66us (737us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321478.366381:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2174 00000100:00080000:3.0:1767321478.366407:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321478.366411:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321478.366414:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985690496:0@lo:38:kworker.0 00010000:00010000:1.0:1767321479.003512:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321480.031761:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321480.923578:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321480.923659:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985691520:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321480.923728:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321480.923791:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691520 00000100:00100000:2.0:1767321480.923800:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691520 00000100:00100000:2.0:1767321480.923872:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3002 00000100:00100000:2.0:1767321480.923882:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985691520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321480.923916:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321480.923966:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985691520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321480.923973:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3002 00000100:00100000:0.0:1767321480.924098:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985691520:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321480.924224:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321480.926009:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985691392:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321480.926079:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321480.933702:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691392 00000100:00100000:0.0:1767321480.933720:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691392 00000100:00100000:0.0:1767321480.933873:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3003 00000100:00100000:0.0:1767321480.933883:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985691392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321480.933920:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321480.934016:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985691392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 133us (7935us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321480.934023:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3003 00000100:00100000:1.0:1767321480.934299:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985691392:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321480.935814:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767321481.052028:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321482.075437:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321483.099613:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321483.434792:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321483.435034:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920675968 00000100:00100000:1.0:1767321483.435049:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920675968 00000100:00100000:1.0:1767321483.435153:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5848 00000100:00100000:1.0:1767321483.435209:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920675968:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321483.435323:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920675968:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 159us (542us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321483.435331:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5848 00000100:00080000:3.0:1767321483.435941:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321483.447885:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321483.451899:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920675840 00000100:00100000:1.0:1767321483.451919:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920675840 00000100:00100000:1.0:1767321483.452005:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6737 00000100:00100000:1.0:1767321483.452014:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920675840:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321483.452039:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321483.452124:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920675840:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 110us (4250us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321483.452132:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6737 00000100:00080000:3.0:1767321483.483846:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321483.483856:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321483.483859:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321483.483863:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321483.483867:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321483.483876:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6726 00000100:00080000:3.0:1767321483.483880:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321483.483886:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321483.484010:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18e300 x1853163985691648/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321483.484075:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985691648:0@lo:38:kworker.0 00000100:00100000:0.0:1767321483.484137:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321483.484289:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321483.484294:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321483.484296:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321483.484300:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6726 00000100:00080000:3.0:1767321483.484545:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321483.484550:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321483.484568:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18ed80 x1853163985691776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321483.484633:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691648 00000100:00100000:1.0:1767321483.484640:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691648 00000100:00100000:1.0:1767321483.484692:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6738 00000100:00100000:1.0:1767321483.484700:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985691648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321483.484724:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321483.485152:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985691776:0@lo:38:kworker.0 00000100:00100000:0.0:1767321483.485615:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321483.485734:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321483.485738:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321483.485741:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321483.485746:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6726 00000100:00080000:3.0:1767321483.485750:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321483.485757:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321483.485778:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06680 x1853163985692032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321483.485807:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321483.485810:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321483.485812:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321483.485815:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6726 00000100:00080000:3.0:1767321483.485817:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321483.485820:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321483.485831:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985692288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321483.485868:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985692416:0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.485918:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321483.485963:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321483.485969:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985692160:0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.485994:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321483.486027:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692160 00000100:00100000:3.0:1767321483.486033:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692160 00000100:00100000:3.0:1767321483.486091:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5849 00000100:00100000:3.0:1767321483.486099:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985692160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486144:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985692160:12345-0@lo:400:kworker.0 Request processed in 46us (148us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321483.486152:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5849 00000100:00080000:3.0:1767321483.486192:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321483.486222:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985692160:0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486471:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985692544:0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486498:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321483.486522:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321483.486527:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985691904:0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486547:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321483.486569:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691904 00000100:00100000:3.0:1767321483.486572:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691904 00000100:00100000:3.0:1767321483.486592:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5850 00000100:00100000:3.0:1767321483.486596:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985691904:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486618:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985691904:12345-0@lo:400:kworker.0 Request processed in 22us (71us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321483.486623:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5850 00000100:00080000:3.0:1767321483.486636:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321483.486653:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985691904:0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486692:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692416 00000100:00100000:3.0:1767321483.486695:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692416 00000100:00100000:3.0:1767321483.486720:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692544 00000100:00100000:3.0:1767321483.486722:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692544 00000100:00100000:3.0:1767321483.486733:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3771 00000100:00100000:3.0:1767321483.486737:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985692416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486763:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985692416:12345-0@lo:400:kworker.0 Request processed in 25us (845us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321483.486769:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3771 00000100:00100000:3.0:1767321483.486780:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3772 00000100:00100000:3.0:1767321483.486785:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985692544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486814:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985692544:12345-0@lo:400:kworker.0 Request processed in 29us (315us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321483.486820:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3772 00000100:00080000:3.0:1767321483.486833:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321483.486835:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321483.486850:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985692416:0@lo:400:kworker.0 00000100:00100000:3.0:1767321483.486866:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985692544:0@lo:400:kworker.0 00000100:00100000:1.0:1767321483.487073:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985691648:12345-0@lo:38:kworker.0 Request processed in 2371us (2933us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321483.487082:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6738 00000100:00080000:0.0:1767321483.487694:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321483.487703:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321483.487707:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985691648:0@lo:38:kworker.0 00000100:00100000:0.0:1767321483.487735:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985692032:0@lo:38:kworker.0 00000100:00100000:0.0:1767321483.487777:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321483.487803:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985692288:0@lo:38:kworker.0 00000100:00100000:1.0:1767321483.487806:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692032 00000100:00100000:1.0:1767321483.487812:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692032 00000100:00100000:0.0:1767321483.487823:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321483.487854:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985691776 00000100:00100000:2.0:1767321483.487859:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985691776 00000100:00100000:2.0:1767321483.487890:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2175 00000100:00100000:2.0:1767321483.487896:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985691776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321483.487916:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321483.487970:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985691776:12345-0@lo:38:kworker.0 Request processed in 72us (2355us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321483.487976:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2175 00000100:00080000:0.0:1767321483.488016:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321483.488022:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321483.488025:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985691776:0@lo:38:kworker.0 00000100:00100000:3.0:1767321483.488145:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692288 00000100:00100000:1.0:1767321483.488148:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6739 00000100:00100000:3.0:1767321483.488149:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692288 00000100:00100000:1.0:1767321483.488152:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985692032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321483.488181:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321483.488215:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db76d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985692032:12345-0@lo:38:kworker.0 Request processed in 61us (437us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321483.488222:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6739 00000100:00100000:3.0:1767321483.488227:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6740 00000100:00100000:3.0:1767321483.488232:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985692288:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767321483.488254:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321483.488258:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321483.488261:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985692032:0@lo:38:kworker.0 00010000:00020000:3.0:1767321483.488465:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321483.488505:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985692288:12345-0@lo:38:kworker.0 Request processed in 271us (680us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321483.488512:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6740 00000100:00080000:1.0:1767321483.488557:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321483.488562:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321483.488565:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985692288:0@lo:38:kworker.0 00010000:00010000:1.0:1767321484.123602:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321485.147473:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321486.043620:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321486.043639:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985692672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321486.043695:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321486.043822:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692672 00000100:00100000:2.0:1767321486.043829:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692672 00000100:00100000:2.0:1767321486.043894:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3004 00000100:00100000:2.0:1767321486.043904:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985692672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321486.043944:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321486.043981:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985692672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (285us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321486.043987:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3004 00000100:00080000:3.0:1767321486.044043:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321486.044251:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f05880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985692672:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321486.044483:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321486.044493:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985692800:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321486.044520:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321486.044591:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692800 00000100:00100000:1.0:1767321486.044596:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692800 00000100:00100000:1.0:1767321486.044641:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3005 00000100:00100000:1.0:1767321486.044646:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985692800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321486.044660:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321486.044683:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985692800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 37us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321486.044687:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3005 00000100:00100000:0.0:1767321486.044773:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3ff5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985692800:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321486.044970:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321486.171452:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321487.198441:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321488.222336:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321488.555888:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321488.555944:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321488.556062:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920676096 00000100:00100000:2.0:1767321488.556073:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920676096 00000100:00100000:2.0:1767321488.556114:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920676224 00000100:00100000:2.0:1767321488.556118:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920676224 00000100:00100000:2.0:1767321488.556212:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5851 00000100:00100000:2.0:1767321488.556223:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920676224:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321488.556313:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920676224:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (367us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321488.556334:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6741 00000100:00100000:3.0:1767321488.556348:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920676096:12345-192.168.204.26@tcp:38:kworker.0 00000100:00080000:0.0:1767321488.556362:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:3.0:1767321488.556381:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321488.556450:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920676096:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 102us (572us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321488.558055:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5851 00000100:00100000:3.0:1767321488.559622:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6741 00000100:00080000:1.0:1767321488.604449:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321488.604463:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321488.604466:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321488.604476:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321488.604482:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321488.604493:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6731 00000100:00080000:1.0:1767321488.604499:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321488.604509:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321488.604573:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcdb100 x1853163985692928/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321488.604610:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321488.604614:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321488.604616:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321488.604620:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6731 00000100:00080000:1.0:1767321488.604622:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321488.604626:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321488.604641:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcda680 x1853163985693056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321488.604691:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985692928:0@lo:38:kworker.0 00000100:00080000:1.0:1767321488.604694:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321488.604707:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321488.604711:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321488.604717:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6731 00000100:00080000:1.0:1767321488.604719:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321488.604724:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321488.604743:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12300 x1853163985693312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321488.604780:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321488.604784:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767321488.604791:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321488.604856:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985693056:0@lo:38:kworker.0 00000100:00080000:1.0:1767321488.604888:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321488.604894:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6731 00000100:00080000:1.0:1767321488.604902:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321488.604908:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321488.604928:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13100 x1853163985693568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321488.604942:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321488.604992:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985693440:0@lo:400:kworker.0 00000100:00100000:0.0:1767321488.605010:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985693312:0@lo:38:kworker.0 00000100:00100000:1.0:1767321488.605050:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321488.605090:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321488.605096:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985693696:0@lo:400:kworker.0 00000100:00100000:1.0:1767321488.605121:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321488.605357:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985693824:0@lo:400:kworker.0 00000100:00100000:3.0:1767321488.605395:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321488.605416:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321488.605430:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985693184:0@lo:400:kworker.0 00000100:00100000:3.0:1767321488.605459:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321488.605498:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693696 00000100:00100000:3.0:1767321488.605506:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693696 00000100:00100000:3.0:1767321488.605576:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693824 00000100:00100000:3.0:1767321488.605580:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693824 00000100:00100000:3.0:1767321488.605603:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3773 00000100:00100000:3.0:1767321488.605613:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985693696:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321488.605672:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985693696:12345-0@lo:400:kworker.0 Request processed in 60us (550us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321488.605680:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3773 00000100:00100000:3.0:1767321488.605697:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3774 00000100:00100000:3.0:1767321488.605702:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985693824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321488.605726:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985693824:12345-0@lo:400:kworker.0 Request processed in 24us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321488.605731:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3774 00000100:00080000:3.0:1767321488.605756:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321488.605760:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321488.605788:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985693824:0@lo:400:kworker.0 00000100:00100000:1.0:1767321488.606048:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985693696:0@lo:400:kworker.0 00000100:00100000:1.0:1767321488.606087:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985692928 00000100:00100000:1.0:1767321488.606091:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985692928 00000100:00100000:1.0:1767321488.606117:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693440 00000100:00100000:1.0:1767321488.606120:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693440 00000100:00080000:3.0:1767321488.606278:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321488.606374:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693184 00000100:00100000:1.0:1767321488.606378:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693184 00000100:00100000:2.0:1767321488.606457:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693056 00000100:00100000:2.0:1767321488.606462:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693056 00000100:00100000:2.0:1767321488.606493:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2176 00000100:00100000:2.0:1767321488.606499:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985693056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321488.606523:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321488.606578:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985693056:12345-0@lo:38:kworker.0 Request processed in 78us (1637us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321488.606584:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2176 00000100:00100000:3.0:1767321488.609484:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5852 00000100:00100000:3.0:1767321488.609518:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985693440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321488.609650:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985693440:12345-0@lo:400:kworker.0 Request processed in 152us (4601us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321488.609657:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5852 00000100:00100000:3.0:1767321488.609682:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5853 00000100:00100000:3.0:1767321488.609688:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985693184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321488.609713:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985693184:12345-0@lo:400:kworker.0 Request processed in 26us (4255us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321488.609717:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5853 00000100:00100000:3.0:1767321488.609726:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6742 00000100:00100000:3.0:1767321488.609730:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985692928:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321488.609750:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321488.609770:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985692928:12345-0@lo:38:kworker.0 Request processed in 39us (4985us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321488.609774:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6742 00000100:00100000:3.0:1767321488.609835:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985693184:0@lo:400:kworker.0 00000100:00080000:3.0:1767321488.609913:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321488.609941:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985693440:0@lo:400:kworker.0 00000100:00100000:3.0:1767321488.610001:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321488.610437:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693312 00000100:00100000:3.0:1767321488.610447:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693312 00000100:00100000:3.0:1767321488.610486:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6743 00000100:00100000:3.0:1767321488.610493:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985693312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321488.610507:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321488.610543:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985693312:12345-0@lo:38:kworker.0 Request processed in 50us (543us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321488.610549:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6743 00000100:00080000:3.0:1767321488.610588:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321488.610594:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321488.610598:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985693312:0@lo:38:kworker.0 00000100:00100000:3.0:1767321488.610609:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985693568:0@lo:38:kworker.0 00000100:00100000:3.0:1767321488.610635:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321488.610654:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693568 00000100:00100000:3.0:1767321488.610657:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693568 00000100:00100000:3.0:1767321488.610674:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6744 00000100:00100000:3.0:1767321488.610678:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985693568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321488.610687:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321488.610709:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985693568:12345-0@lo:38:kworker.0 Request processed in 29us (73us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321488.610715:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6744 00000100:00080000:3.0:1767321488.610737:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321488.610741:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321488.610744:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985693568:0@lo:38:kworker.0 00000100:00080000:3.0:1767321488.610757:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321488.610761:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321488.610763:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985692928:0@lo:38:kworker.0 00000100:00080000:3.0:1767321488.610778:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321488.610781:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321488.610784:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985693056:0@lo:38:kworker.0 00010000:00010000:2.0:1767321489.244443:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321490.267626:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321491.163376:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321491.163414:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985693952:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321491.163433:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321491.163448:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985694080:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321491.166230:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321491.166341:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985693952 00000100:00100000:0.0:1767321491.166358:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985693952 00000100:00100000:0.0:1767321491.166466:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3006 00000100:00100000:0.0:1767321491.166479:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985693952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321491.166522:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:2.0:1767321491.166617:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321491.167609:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321491.167730:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985694080 00000100:00100000:2.0:1767321491.167739:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985694080 00000100:00100000:2.0:1767321491.167807:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3007 00000100:00100000:2.0:1767321491.167847:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985694080:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321491.167877:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321491.167938:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985694080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (333us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321491.167947:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3007 00000100:00100000:1.0:1767321491.168069:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985694080:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321491.169193:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321491.170087:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985693952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3603us (3860us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321491.170104:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3006 00000100:00100000:3.0:1767321491.170278:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985693952:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321491.297412:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321492.318578:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321492.653602:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321492.660349:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920676608 00000100:00100000:1.0:1767321492.660366:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920676608 00000100:00100000:1.0:1767321492.660463:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5854 00000100:00100000:1.0:1767321492.660477:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920676608:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321492.660571:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920676608:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (6979us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321492.660580:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5854 00000100:00080000:3.0:1767321492.660858:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767321492.661984:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920676352 offset 0 length 520: 4 00000100:00100000:0.0:1767321492.662023:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321492.662101:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920676480 00000100:00100000:3.0:1767321492.662111:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920676480 00000100:00100000:3.0:1767321492.662190:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6745 00000100:00100000:3.0:1767321492.662199:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920676480:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321492.662222:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321492.662388:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920676480:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 187us (362us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321492.662396:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6745 00010000:00010000:0.0:1767321493.340628:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321493.723314:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321468/real 1767321468] req@ffff8ce1bbe22680 x1853163985687680/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321493 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321493.723349:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1bbe22680 x1853163985687680/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321493 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321493.723369:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321493.723391:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321493.723395:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985687680:0@lo:250:kworker.0 00000100:00080000:1.0:1767321493.723701:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321493.723708:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767321493.723711:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321493.723717:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6716 00000100:00080000:1.0:1767321493.723722:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321493.723730:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321493.723797:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a11880 x1853163985694208/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321493.723827:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321493.723831:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321493.723833:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321493.723837:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6736 00000100:00080000:1.0:1767321493.723840:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321493.723844:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321493.723860:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10700 x1853163985694336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321493.723868:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321493.723871:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321493.723872:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321493.723875:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6736 00000100:00080000:1.0:1767321493.723878:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321493.723882:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321493.723891:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a10e00 x1853163985694464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321493.723948:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321493.723952:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321493.723954:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321493.723958:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6736 00000100:00080000:1.0:1767321493.723960:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321493.723963:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321493.723974:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a13800 x1853163985694720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321493.724008:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321493.724011:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321493.724013:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321493.724016:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6736 00000100:00080000:1.0:1767321493.724019:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321493.724022:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321493.724035:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce184a12680 x1853163985694976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321493.724073:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985695104:0@lo:400:kworker.0 00000100:00100000:1.0:1767321493.724134:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321493.724211:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321493.724218:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985694848:0@lo:400:kworker.0 00000100:00100000:1.0:1767321493.724245:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321493.724550:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985695232:0@lo:400:kworker.0 00000100:00100000:0.0:1767321493.724581:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321493.724610:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321493.724615:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985694592:0@lo:400:kworker.0 00000100:00100000:0.0:1767321493.724637:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321493.724863:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985695104 00000100:00100000:3.0:1767321493.724871:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985695104 00000100:00100000:3.0:1767321493.724939:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985695232 00000100:00100000:3.0:1767321493.724942:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985695232 00000100:00100000:3.0:1767321493.724968:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3775 00000100:00100000:3.0:1767321493.724976:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985695104:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321493.725044:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985695104:12345-0@lo:400:kworker.0 Request processed in 69us (910us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321493.725052:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3775 00000100:00100000:3.0:1767321493.725070:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3776 00000100:00100000:3.0:1767321493.725076:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985695232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321493.725105:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985695232:12345-0@lo:400:kworker.0 Request processed in 29us (523us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321493.725111:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3776 00000100:00100000:3.0:1767321493.725132:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985694848 00000100:00100000:3.0:1767321493.725135:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985694848 00000100:00100000:3.0:1767321493.725157:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985694592 00000100:00100000:3.0:1767321493.725160:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985694592 00000100:00100000:3.0:1767321493.725203:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5855 00000100:00100000:3.0:1767321493.725209:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985694848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321493.725239:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985694848:12345-0@lo:400:kworker.0 Request processed in 31us (994us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321493.725244:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5855 00000100:00100000:3.0:1767321493.725257:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5856 00000100:00100000:3.0:1767321493.725263:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985694592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321493.725288:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985694592:12345-0@lo:400:kworker.0 Request processed in 26us (650us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321493.725294:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5856 00000100:00100000:3.0:1767321493.725335:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985694848:0@lo:400:kworker.0 00000100:00100000:3.0:1767321493.725344:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985695104:0@lo:400:kworker.0 00000100:00100000:3.0:1767321493.725368:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985694208:0@lo:250:kworker.0 00000400:00000100:3.0:1767321493.725386:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985694208 offset 0 length 520: 4 00000100:00100000:3.0:1767321493.725400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985694336:0@lo:38:kworker.0 00000100:00100000:3.0:1767321493.725415:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321493.725425:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985694464:0@lo:38:kworker.0 00000100:00100000:3.0:1767321493.725438:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321493.725447:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985694720:0@lo:38:kworker.0 00000100:00100000:3.0:1767321493.725457:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321493.725465:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985694976:0@lo:38:kworker.0 00000100:00100000:3.0:1767321493.725476:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321493.725494:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985694464 00000100:00100000:3.0:1767321493.725496:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985694464 00000100:00100000:3.0:1767321493.725510:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2177 00000100:00100000:3.0:1767321493.725515:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985694464:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321493.725532:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321493.725565:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985694464:12345-0@lo:38:kworker.0 Request processed in 49us (124us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321493.725570:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2177 00000100:00100000:3.0:1767321493.725587:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985694336 00000100:00100000:3.0:1767321493.725590:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985694336 00000100:00100000:3.0:1767321493.725601:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985694720 00000100:00100000:3.0:1767321493.725603:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985694720 00000100:00100000:3.0:1767321493.725608:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985694976 00000100:00100000:3.0:1767321493.725610:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985694976 00000100:00100000:3.0:1767321493.725618:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6746 00000100:00100000:3.0:1767321493.725622:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985694336:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321493.725631:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321493.725657:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985694336:12345-0@lo:38:kworker.0 Request processed in 33us (240us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321493.725663:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6746 00000100:00100000:3.0:1767321493.725687:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6747 00000100:00100000:3.0:1767321493.725691:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985694720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321493.725701:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321493.725722:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985694720:12345-0@lo:38:kworker.0 Request processed in 30us (263us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321493.725727:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6747 00000100:00100000:3.0:1767321493.725738:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6748 00000100:00100000:3.0:1767321493.725742:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985694976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321493.725750:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321493.725781:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985694976:12345-0@lo:38:kworker.0 Request processed in 37us (303us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321493.725787:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6748 00000100:00080000:3.0:1767321493.725813:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321493.725818:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321493.725821:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985694336:0@lo:38:kworker.0 00000100:00080000:3.0:1767321493.725829:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321493.725833:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321493.725835:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a10e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985694464:0@lo:38:kworker.0 00000100:00080000:3.0:1767321493.725842:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321493.725845:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321493.725846:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985694720:0@lo:38:kworker.0 00000100:00080000:3.0:1767321493.725851:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321493.725854:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321493.725855:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985694976:0@lo:38:kworker.0 00000100:00080000:2.0:1767321493.725985:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321493.725998:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321493.726000:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321493.726185:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985694592:0@lo:400:kworker.0 00000100:00100000:0.0:1767321493.726211:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985695232:0@lo:400:kworker.0 00010000:00010000:1.0:1767321494.365229:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321495.387390:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321496.284401:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321496.284444:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985695488:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321496.284542:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321496.284620:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321496.284630:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985695360:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321496.284665:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321496.284756:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985695488 00000100:00100000:0.0:1767321496.284766:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985695488 00000100:00100000:0.0:1767321496.284836:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985695360 00000100:00100000:0.0:1767321496.284839:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985695360 00000100:00100000:0.0:1767321496.284862:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3008 00000100:00100000:0.0:1767321496.284870:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985695488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321496.284919:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321496.284963:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985695488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (421us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321496.284970:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3008 00000100:00100000:0.0:1767321496.284987:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3009 00000100:00100000:0.0:1767321496.284993:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985695360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321496.285009:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321496.285037:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985695360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321496.285043:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3009 00000100:00080000:3.0:1767321496.285148:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321496.285152:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321496.285211:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a12300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985695360:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321496.285245:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985695488:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321496.411455:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321497.435419:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321497.775286:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321497.775332:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321497.775396:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920676736 00000100:00100000:0.0:1767321497.775409:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920676736 00000100:00100000:0.0:1767321497.775454:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920676864 00000100:00100000:0.0:1767321497.775457:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920676864 00000100:00100000:0.0:1767321497.775499:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5857 00000100:00100000:0.0:1767321497.775508:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920676864:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321497.775638:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6749 00000100:00100000:2.0:1767321497.775644:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920676736:12345-192.168.204.26@tcp:38:kworker.0 00000100:00080000:3.0:1767321497.775776:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:2.0:1767321497.775876:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321497.776015:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920676736:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 369us (739us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321497.776024:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6749 00000100:00100000:0.0:1767321497.785291:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920676864:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 9770us (9943us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321497.785307:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5857 00010000:00010000:2.0:1767321498.460100:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321498.844054:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321498.844066:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321498.844069:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321498.844073:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321498.844078:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321498.844086:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6741 00000100:00080000:3.0:1767321498.844092:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321498.844100:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321498.844157:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06a00 x1853163985695616/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321498.844200:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321498.844204:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321498.844205:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321498.844209:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6741 00000100:00080000:3.0:1767321498.844211:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321498.844214:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321498.844227:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188f06680 x1853163985695744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321498.844305:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321498.844390:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985695616:0@lo:38:kworker.0 00000100:00100000:0.0:1767321498.844453:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321498.844512:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985695744:0@lo:38:kworker.0 00000100:00100000:0.0:1767321498.844537:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321498.844579:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985695616 00000100:00100000:0.0:1767321498.844587:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985695616 00000100:00100000:0.0:1767321498.844623:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6750 00000100:00100000:0.0:1767321498.844630:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985695616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321498.844650:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321498.844717:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985695616:12345-0@lo:38:kworker.0 Request processed in 86us (263us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321498.844728:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6750 00000100:00080000:0.0:1767321498.844777:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321498.844782:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321498.844786:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985695616:0@lo:38:kworker.0 00000100:00100000:0.0:1767321498.844825:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985695744 00000100:00100000:0.0:1767321498.844829:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985695744 00000100:00100000:0.0:1767321498.844852:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2178 00000100:00100000:0.0:1767321498.844856:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985695744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321498.844866:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321498.844895:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985695744:12345-0@lo:38:kworker.0 Request processed in 37us (357us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321498.844901:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2178 00000100:00080000:0.0:1767321498.844924:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321498.844927:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321498.844931:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985695744:0@lo:38:kworker.0 00000100:00080000:3.0:1767321498.845051:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321498.845056:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321498.845058:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321498.845062:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6741 00000100:00080000:3.0:1767321498.845065:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321498.845069:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321498.845085:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcda680 x1853163985696000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321498.845129:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321498.845132:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321498.845134:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321498.845137:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6741 00000100:00080000:3.0:1767321498.845140:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321498.845143:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321498.845154:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd9180 x1853163985696256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321498.845233:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985696128:0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.845264:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321498.845289:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321498.845297:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985696384:0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.845323:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321498.845350:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696384 00000100:00100000:1.0:1767321498.845354:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696384 00000100:00100000:1.0:1767321498.845395:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3777 00000100:00100000:1.0:1767321498.845401:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985696384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.845439:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985696384:12345-0@lo:400:kworker.0 Request processed in 38us (114us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321498.845445:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3777 00000100:00080000:1.0:1767321498.845462:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321498.845474:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696128 00000100:00100000:1.0:1767321498.845478:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696128 00000100:00100000:1.0:1767321498.845505:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5858 00000100:00100000:1.0:1767321498.845512:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985696128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.845542:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985696128:12345-0@lo:400:kworker.0 Request processed in 32us (277us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321498.845548:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5858 00000100:00080000:1.0:1767321498.845561:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321498.845579:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985696384:0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.845588:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985696128:0@lo:400:kworker.0 00000100:00100000:0.0:1767321498.845972:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985696000:0@lo:38:kworker.0 00000100:00100000:0.0:1767321498.846001:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321498.846028:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985696256:0@lo:38:kworker.0 00000100:00100000:0.0:1767321498.846046:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321498.846071:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696000 00000100:00100000:0.0:1767321498.846075:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696000 00000100:00100000:0.0:1767321498.846091:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696256 00000100:00100000:0.0:1767321498.846094:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696256 00000100:00100000:0.0:1767321498.846108:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6751 00000100:00100000:0.0:1767321498.846112:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985696000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321498.846126:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321498.846185:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985696000:12345-0@lo:38:kworker.0 Request processed in 50us (162us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321498.846191:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6751 00000100:00100000:0.0:1767321498.846204:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6752 00000100:00100000:0.0:1767321498.846207:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985696256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321498.846217:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321498.846242:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985696256:12345-0@lo:38:kworker.0 Request processed in 33us (194us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321498.846247:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6752 00000100:00080000:0.0:1767321498.846276:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321498.846280:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321498.846283:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985696000:0@lo:38:kworker.0 00000100:00080000:0.0:1767321498.846291:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321498.846294:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321498.846297:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985696256:0@lo:38:kworker.0 00000100:00100000:2.0:1767321498.849775:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985696512:0@lo:400:kworker.0 00000100:00100000:2.0:1767321498.849874:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321498.850006:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985695872:0@lo:400:kworker.0 00000100:00100000:2.0:1767321498.850035:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321498.850653:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985695872 00000100:00100000:0.0:1767321498.850680:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985695872 00000100:00100000:0.0:1767321498.850772:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5859 00000100:00100000:0.0:1767321498.850783:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985695872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321498.850844:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985695872:12345-0@lo:400:kworker.0 Request processed in 63us (807us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321498.850852:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5859 00000100:00080000:2.0:1767321498.851435:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321498.851470:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985695872:0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.854441:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696512 00000100:00100000:1.0:1767321498.854457:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696512 00000100:00100000:1.0:1767321498.854548:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3778 00000100:00100000:1.0:1767321498.854558:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7a01450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985696512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.854688:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7a01450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985696512:12345-0@lo:400:kworker.0 Request processed in 132us (4813us total) trans 0 rc 0/0 00000100:00080000:2.0:1767321498.854952:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321498.855157:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985696512:0@lo:400:kworker.0 00000100:00100000:1.0:1767321498.855439:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3778 00010000:00010000:0.0:1767321499.484216:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321500.507730:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321501.403485:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321501.403609:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985696640:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321501.403704:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321501.403834:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696640 00000100:00100000:0.0:1767321501.403843:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696640 00000100:00100000:0.0:1767321501.403913:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3010 00000100:00100000:0.0:1767321501.403924:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985696640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321501.403953:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321501.404205:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985696640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 282us (499us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321501.404213:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3010 00000100:00080000:3.0:1767321501.404283:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321501.404325:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985696640:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321501.404452:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321501.404461:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985696768:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321501.404489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321501.404613:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696768 00000100:00100000:2.0:1767321501.404620:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696768 00000100:00080000:3.0:1767321501.404654:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321501.404700:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3011 00000100:00100000:2.0:1767321501.404710:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985696768:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321501.404738:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321501.404785:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985696768:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321501.404791:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3011 00000100:00100000:1.0:1767321501.405064:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985696768:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321501.533062:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321502.555438:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321502.891669:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321502.891870:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920677120 00000100:00100000:0.0:1767321502.891892:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920677120 00000100:00100000:0.0:1767321502.891994:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5860 00000100:00100000:0.0:1767321502.892006:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920677120:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321502.892112:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920677120:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 108us (464us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321502.892120:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5860 00000100:00080000:3.0:1767321502.892873:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321502.893704:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321502.893993:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920676992 00000100:00100000:1.0:1767321502.894006:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920676992 00000100:00100000:1.0:1767321502.894065:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6753 00000100:00100000:1.0:1767321502.894075:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920676992:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321502.894107:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321502.894229:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920676992:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 154us (532us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321502.894237:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6753 00010000:00010000:1.0:1767321503.581924:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321503.964066:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321503.964087:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321503.964091:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321503.964097:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321503.964103:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321503.964118:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6747 00000100:00080000:0.0:1767321503.964126:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321503.964138:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321503.964230:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6680 x1853163985696896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321503.964252:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321503.964256:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321503.964258:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321503.964261:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6747 00000100:00080000:0.0:1767321503.964264:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321503.964268:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321503.964281:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6300 x1853163985697024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321503.964324:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321503.964328:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321503.964330:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321503.964334:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6747 00000100:00080000:0.0:1767321503.964337:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321503.964340:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321503.964353:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985697280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321503.964684:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985696896:0@lo:38:kworker.0 00000100:00100000:2.0:1767321503.964793:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321503.964877:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985697024:0@lo:38:kworker.0 00000100:00100000:2.0:1767321503.964918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321503.964948:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985697280:0@lo:38:kworker.0 00000100:00100000:2.0:1767321503.964971:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321503.965032:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697024 00000100:00100000:2.0:1767321503.965045:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697024 00000100:00100000:2.0:1767321503.965099:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2179 00000100:00100000:2.0:1767321503.965109:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985697024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321503.965141:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321503.965225:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985697024:12345-0@lo:38:kworker.0 Request processed in 116us (305us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321503.965234:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2179 00000100:00100000:2.0:1767321503.965268:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985696896 00000100:00100000:2.0:1767321503.965271:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985696896 00000100:00100000:2.0:1767321503.965295:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697280 00000100:00100000:2.0:1767321503.965298:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697280 00000100:00100000:2.0:1767321503.965319:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6754 00000100:00100000:2.0:1767321503.965323:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985696896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321503.965334:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321503.965364:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985696896:12345-0@lo:38:kworker.0 Request processed in 40us (572us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321503.965381:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6754 00000100:00100000:2.0:1767321503.965394:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6755 00000100:00100000:2.0:1767321503.965398:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985697280:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321503.965407:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321503.965774:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321503.965790:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985697152:0@lo:400:kworker.0 00000100:00100000:3.0:1767321503.965832:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321503.965877:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697152 00000100:00100000:3.0:1767321503.965882:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697152 00000100:00100000:3.0:1767321503.965958:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5861 00000100:00100000:3.0:1767321503.965967:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985697152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321503.966032:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985697152:12345-0@lo:400:kworker.0 Request processed in 67us (200us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321503.966040:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5861 00000100:00080000:3.0:1767321503.966065:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321503.966117:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985697152:0@lo:400:kworker.0 00000100:00080000:3.0:1767321503.966213:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321503.966221:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321503.966225:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985696896:0@lo:38:kworker.0 00000100:00080000:3.0:1767321503.966236:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321503.966241:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321503.966244:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985697024:0@lo:38:kworker.0 00000100:00100000:2.0:1767321503.966867:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985697280:12345-0@lo:38:kworker.0 Request processed in 1461us (1889us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321503.966879:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6755 00000100:00080000:0.0:1767321503.968188:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321503.968199:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321503.968203:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321503.968213:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6747 00000100:00080000:0.0:1767321503.968220:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321503.968229:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321503.968287:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6300 x1853163985697536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321503.968397:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985697664:0@lo:400:kworker.0 00000100:00100000:1.0:1767321503.968644:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321503.968660:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985697792:0@lo:400:kworker.0 00000100:00100000:2.0:1767321503.968727:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985697408:0@lo:400:kworker.0 00000100:00100000:3.0:1767321503.968794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985697536:0@lo:38:kworker.0 00000100:00100000:1.0:1767321503.971734:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321503.971819:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697792 00000100:00100000:1.0:1767321503.971829:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697792 00000100:00100000:1.0:1767321503.971924:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3779 00000100:00100000:1.0:1767321503.971933:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985697792:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321503.971995:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985697792:12345-0@lo:400:kworker.0 Request processed in 63us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321503.972001:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3779 00000100:00080000:1.0:1767321503.972027:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321503.972068:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985697792:0@lo:400:kworker.0 00000100:00100000:2.0:1767321503.972554:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321503.972787:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697408 00000100:00100000:1.0:1767321503.972792:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697408 00000100:00100000:1.0:1767321503.972834:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5862 00000100:00100000:1.0:1767321503.972842:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985697408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321503.972888:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985697408:12345-0@lo:400:kworker.0 Request processed in 46us (334us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321503.972894:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5862 00000100:00080000:2.0:1767321503.972943:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321503.972970:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985697408:0@lo:400:kworker.0 00000100:00100000:3.0:1767321503.973143:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321503.973212:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321503.973217:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321503.973220:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985697280:0@lo:38:kworker.0 00000100:00100000:2.0:1767321503.973359:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697536 00000100:00100000:2.0:1767321503.973362:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697536 00000100:00100000:2.0:1767321503.973506:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6756 00000100:00100000:2.0:1767321503.973513:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985697536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321503.973547:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321503.973599:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985697536:12345-0@lo:38:kworker.0 Request processed in 85us (455us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321503.973606:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6756 00000100:00080000:3.0:1767321503.973662:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321503.973665:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321503.973667:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985697536:0@lo:38:kworker.0 00000100:00100000:0.0:1767321503.974967:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321503.975319:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697664 00000100:00100000:1.0:1767321503.975330:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697664 00000100:00080000:2.0:1767321503.975446:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321503.975545:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3780 00000100:00100000:3.0:1767321503.975555:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985697664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321503.975640:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985697664:12345-0@lo:400:kworker.0 Request processed in 87us (679us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321503.975648:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3780 00000100:00100000:0.0:1767321503.976067:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985697664:0@lo:400:kworker.0 00010000:00010000:2.0:1767321504.606934:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321505.627483:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321506.524154:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985697920:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321506.524235:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321506.524261:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985698048:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321506.524380:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321506.524421:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321506.524916:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698048 00000100:00100000:3.0:1767321506.524929:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698048 00000100:00080000:2.0:1767321506.525144:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321506.525146:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985697920 00000100:00100000:1.0:1767321506.525153:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985697920 00000100:00100000:3.0:1767321506.525445:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3012 00000100:00100000:3.0:1767321506.525457:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985698048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321506.525499:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321506.525608:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985698048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 154us (1239us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321506.525617:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3012 00000100:00100000:2.0:1767321506.525665:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985698048:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321506.525801:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321506.525832:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3013 00000100:00100000:1.0:1767321506.525841:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985697920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321506.525862:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321506.525912:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985697920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (1491us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321506.525918:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3013 00000100:00100000:2.0:1767321506.525996:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985697920:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321506.654351:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321507.675538:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321508.015647:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321508.015715:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321508.015804:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920677248 00000100:00100000:1.0:1767321508.015818:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920677248 00000100:00100000:1.0:1767321508.015876:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920677376 00000100:00100000:1.0:1767321508.015880:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920677376 00000100:00100000:1.0:1767321508.015940:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5863 00000100:00100000:1.0:1767321508.015952:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920677376:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321508.016050:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920677376:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 99us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321508.016059:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5863 00000100:00100000:1.0:1767321508.016081:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6757 00000100:00100000:1.0:1767321508.016085:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920677248:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321508.016110:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321508.016149:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920677248:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (523us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321508.016157:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6757 00000100:00080000:2.0:1767321508.016198:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321508.703629:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321509.083336:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321509.083353:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321509.083358:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321509.083363:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321509.083369:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321509.083380:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6752 00000100:00080000:3.0:1767321509.083386:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321509.083396:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321509.083458:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18dc00 x1853163985698176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321509.083490:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321509.083494:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321509.083496:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321509.083500:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6752 00000100:00080000:3.0:1767321509.083502:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321509.083507:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321509.083521:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af18e300 x1853163985698304/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321509.083566:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321509.083570:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321509.083572:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321509.083586:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6752 00000100:00080000:3.0:1767321509.083591:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321509.083595:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321509.083617:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9180 x1853163985698560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321509.083618:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985698432:0@lo:400:kworker.0 00000100:00080000:3.0:1767321509.083643:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321509.083646:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321509.083648:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321509.083652:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6752 00000100:00080000:3.0:1767321509.083654:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321509.083658:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321509.083669:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfbb80 x1853163985698816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321509.083726:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321509.083801:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321509.083808:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985699072:0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.083838:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321509.083874:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698432 00000100:00100000:0.0:1767321509.083884:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698432 00000100:00100000:0.0:1767321509.083964:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5864 00000100:00100000:0.0:1767321509.083977:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985698432:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084044:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985698432:12345-0@lo:400:kworker.0 Request processed in 70us (318us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321509.084052:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5864 00000100:00100000:0.0:1767321509.084102:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985698688:0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084127:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321509.084144:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321509.084150:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985698944:0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084200:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321509.084224:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698688 00000100:00100000:0.0:1767321509.084227:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698688 00000100:00100000:0.0:1767321509.084252:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5865 00000100:00100000:0.0:1767321509.084258:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985698688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084292:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985698688:12345-0@lo:400:kworker.0 Request processed in 33us (165us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321509.084297:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5865 00000100:00100000:0.0:1767321509.084337:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985698688:0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084367:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985698432:0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084408:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699072 00000100:00100000:0.0:1767321509.084671:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699072 00000100:00100000:0.0:1767321509.084711:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698944 00000100:00100000:0.0:1767321509.084715:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698944 00000100:00100000:0.0:1767321509.084736:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3781 00000100:00100000:0.0:1767321509.084743:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985699072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084783:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985699072:12345-0@lo:400:kworker.0 Request processed in 40us (943us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321509.084790:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3781 00000100:00100000:0.0:1767321509.084800:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3782 00000100:00100000:0.0:1767321509.084807:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985698944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084836:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985698944:12345-0@lo:400:kworker.0 Request processed in 29us (635us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321509.084841:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3782 00000100:00100000:0.0:1767321509.084868:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985698944:0@lo:400:kworker.0 00000100:00100000:0.0:1767321509.084891:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985699072:0@lo:400:kworker.0 00000100:00100000:2.0:1767321509.086581:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985698176:0@lo:38:kworker.0 00000100:00100000:2.0:1767321509.086670:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321509.086731:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985698304:0@lo:38:kworker.0 00000100:00100000:2.0:1767321509.086762:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321509.086806:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985698560:0@lo:38:kworker.0 00000100:00100000:2.0:1767321509.086827:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321509.086847:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985698816:0@lo:38:kworker.0 00000100:00100000:2.0:1767321509.086867:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321509.086896:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321509.086902:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321509.086905:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321509.086920:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698176 00000100:00100000:0.0:1767321509.086922:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698560 00000100:00100000:0.0:1767321509.086929:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698560 00000100:00100000:2.0:1767321509.086929:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698176 00000100:00100000:1.0:1767321509.090452:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698304 00000100:00100000:1.0:1767321509.090474:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698304 00000100:00100000:1.0:1767321509.090577:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2180 00000100:00100000:1.0:1767321509.090587:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985698304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321509.090618:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321509.090932:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985698304:12345-0@lo:38:kworker.0 Request processed in 345us (4167us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321509.090941:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2180 00000100:00100000:0.0:1767321509.095856:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985698816 00000100:00100000:0.0:1767321509.095880:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985698816 00000100:00100000:0.0:1767321509.095973:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6758 00000100:00100000:0.0:1767321509.095984:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985698560:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321509.096017:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321509.096143:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985698560:12345-0@lo:38:kworker.0 Request processed in 158us (9312us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321509.096151:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6758 00000100:00100000:0.0:1767321509.096191:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6759 00000100:00100000:0.0:1767321509.096197:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985698176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321509.096209:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321509.096237:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985698176:12345-0@lo:38:kworker.0 Request processed in 39us (9567us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321509.096243:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6759 00000100:00100000:0.0:1767321509.096259:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6760 00000100:00100000:0.0:1767321509.096264:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985698816:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321509.096273:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321509.096297:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985698816:12345-0@lo:38:kworker.0 Request processed in 32us (9429us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321509.096303:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6760 00000100:00080000:0.0:1767321509.096391:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321509.096398:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321509.096405:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985698176:0@lo:38:kworker.0 00000100:00080000:0.0:1767321509.096419:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321509.096423:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321509.096426:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985698304:0@lo:38:kworker.0 00000100:00080000:0.0:1767321509.096439:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321509.096442:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321509.096445:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985698560:0@lo:38:kworker.0 00000100:00080000:0.0:1767321509.096454:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321509.096457:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321509.096461:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985698816:0@lo:38:kworker.0 00010000:00010000:1.0:1767321509.723856:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321510.747428:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321511.644799:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321511.644833:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985699200:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321511.644937:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321511.645236:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699200 00000100:00100000:3.0:1767321511.645247:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699200 00000100:00100000:3.0:1767321511.645323:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3014 00000100:00100000:3.0:1767321511.645334:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985699200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321511.645373:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321511.645709:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985699200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 376us (770us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321511.645719:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3014 00000100:00080000:2.0:1767321511.645814:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321511.645865:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985699200:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321511.646622:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321511.646642:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985699328:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321511.646705:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321511.646800:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699328 00000100:00100000:1.0:1767321511.646809:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699328 00000100:00100000:1.0:1767321511.646888:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3015 00000100:00100000:1.0:1767321511.646898:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985699328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321511.646928:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321511.646988:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985699328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321511.646996:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3015 00000100:00080000:2.0:1767321511.647054:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321511.649959:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985699328:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767321511.772604:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321512.795707:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321513.130893:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321513.130947:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321513.131039:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920677632 00000100:00100000:3.0:1767321513.131055:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920677632 00000100:00100000:3.0:1767321513.131142:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920677504 00000100:00100000:3.0:1767321513.131145:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920677504 00000100:00100000:3.0:1767321513.131221:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5866 00000100:00100000:3.0:1767321513.131235:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920677632:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321513.131343:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920677632:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 112us (470us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321513.131352:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5866 00000100:00100000:3.0:1767321513.131373:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6761 00000100:00100000:3.0:1767321513.131378:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920677504:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321513.131405:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321513.131442:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920677504:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 62us (494us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321513.131448:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6761 00000100:00080000:2.0:1767321513.132352:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321513.820342:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321514.203461:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321514.203475:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321514.203479:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321514.203485:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321514.203491:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321514.203502:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6757 00000100:00080000:3.0:1767321514.203508:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321514.203516:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321514.203573:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfad80 x1853163985699456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321514.203615:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321514.203618:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321514.203620:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321514.203624:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6757 00000100:00080000:3.0:1767321514.203628:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321514.203632:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321514.203647:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb100 x1853163985699584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321514.203686:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321514.203691:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321514.203692:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321514.203696:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6757 00000100:00080000:3.0:1767321514.203698:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321514.203702:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321514.203716:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9c00 x1853163985699840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321514.203741:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321514.203745:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321514.203747:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321514.203751:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6757 00000100:00080000:3.0:1767321514.203753:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321514.203757:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321514.203787:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9f80 x1853163985700096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321514.204214:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985700352:0@lo:400:kworker.0 00000100:00100000:2.0:1767321514.204317:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321514.204403:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321514.204410:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985699712:0@lo:400:kworker.0 00000100:00100000:2.0:1767321514.204440:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321514.204473:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985699968:0@lo:400:kworker.0 00000100:00100000:2.0:1767321514.204496:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321514.204512:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321514.204516:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985700224:0@lo:400:kworker.0 00000100:00100000:2.0:1767321514.204534:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321514.204566:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985700352 00000100:00100000:2.0:1767321514.204575:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985700352 00000100:00100000:2.0:1767321514.204637:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985700224 00000100:00100000:2.0:1767321514.204641:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985700224 00000100:00100000:2.0:1767321514.204665:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3783 00000100:00100000:2.0:1767321514.204677:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985700352:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321514.204738:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985700352:12345-0@lo:400:kworker.0 Request processed in 63us (420us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321514.204746:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3783 00000100:00100000:2.0:1767321514.204766:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3784 00000100:00100000:2.0:1767321514.204870:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985700224:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321514.204906:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985700224:12345-0@lo:400:kworker.0 Request processed in 134us (370us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321514.204913:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3784 00000100:00080000:2.0:1767321514.204932:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321514.204937:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321514.204969:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985700224:0@lo:400:kworker.0 00000100:00100000:2.0:1767321514.204996:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985700352:0@lo:400:kworker.0 00000100:00100000:0.0:1767321514.205450:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985699456:0@lo:38:kworker.0 00000100:00100000:0.0:1767321514.205492:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321514.205523:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985699584:0@lo:38:kworker.0 00000100:00100000:0.0:1767321514.205547:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321514.205572:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985699840:0@lo:38:kworker.0 00000100:00100000:0.0:1767321514.205606:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321514.205623:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985700096:0@lo:38:kworker.0 00000100:00100000:0.0:1767321514.205645:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321514.205680:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699712 00000100:00100000:0.0:1767321514.205686:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699712 00000100:00100000:0.0:1767321514.205732:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699968 00000100:00100000:0.0:1767321514.205735:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699968 00000100:00100000:0.0:1767321514.205748:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699456 00000100:00100000:0.0:1767321514.205751:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699456 00000100:00100000:0.0:1767321514.205760:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699840 00000100:00100000:0.0:1767321514.205762:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699840 00000100:00100000:0.0:1767321514.205769:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985700096 00000100:00100000:0.0:1767321514.205782:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985700096 00000100:00100000:0.0:1767321514.205795:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5867 00000100:00100000:0.0:1767321514.205802:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985699712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321514.205844:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985699712:12345-0@lo:400:kworker.0 Request processed in 43us (1403us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321514.205851:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5867 00000100:00100000:0.0:1767321514.205867:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5868 00000100:00100000:0.0:1767321514.205872:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985699968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321514.205904:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985699968:12345-0@lo:400:kworker.0 Request processed in 31us (1407us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321514.205910:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5868 00000100:00100000:0.0:1767321514.205921:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6762 00000100:00100000:0.0:1767321514.205925:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985699456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321514.205948:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321514.205979:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985699456:12345-0@lo:38:kworker.0 Request processed in 52us (487us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321514.205986:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6762 00000100:00100000:0.0:1767321514.205997:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6763 00000100:00100000:0.0:1767321514.206001:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985699840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321514.206010:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321514.206034:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985699840:12345-0@lo:38:kworker.0 Request processed in 31us (427us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321514.206040:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6763 00000100:00100000:0.0:1767321514.206052:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6764 00000100:00100000:0.0:1767321514.206056:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985700096:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321514.206065:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321514.206087:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985700096:12345-0@lo:38:kworker.0 Request processed in 30us (442us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321514.206093:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6764 00000100:00100000:0.0:1767321514.206117:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985699584 00000100:00100000:0.0:1767321514.206121:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985699584 00000100:00100000:0.0:1767321514.206141:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2181 00000100:00100000:0.0:1767321514.206145:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985699584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321514.206153:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321514.206206:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985699584:12345-0@lo:38:kworker.0 Request processed in 60us (658us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321514.206212:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2181 00000100:00080000:0.0:1767321514.206236:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321514.206264:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985699968:0@lo:400:kworker.0 00000100:00080000:0.0:1767321514.206293:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321514.206298:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321514.206302:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985699456:0@lo:38:kworker.0 00000100:00080000:0.0:1767321514.206310:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321514.206313:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321514.206316:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985699584:0@lo:38:kworker.0 00000100:00080000:0.0:1767321514.206324:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321514.206327:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321514.206330:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985699840:0@lo:38:kworker.0 00000100:00080000:0.0:1767321514.206338:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321514.206341:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321514.206344:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985700096:0@lo:38:kworker.0 00000100:00100000:2.0:1767321514.208370:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985699712:0@lo:400:kworker.0 00010000:00010000:1.0:1767321514.843614:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321515.867523:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321516.763917:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321516.763951:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985700480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321516.764045:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321516.764140:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985700608:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321516.764191:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321516.764463:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985700480 00000100:00100000:0.0:1767321516.764475:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985700480 00000100:00100000:0.0:1767321516.764562:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985700608 00000100:00100000:0.0:1767321516.764566:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985700608 00000100:00100000:0.0:1767321516.764598:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3016 00000100:00100000:0.0:1767321516.764609:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985700480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321516.764649:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321516.764721:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985700480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 114us (675us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321516.764739:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3016 00000100:00100000:0.0:1767321516.764760:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3017 00000100:00100000:0.0:1767321516.764766:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985700608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321516.764783:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321516.764815:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985700608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (624us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321516.764822:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3017 00000100:00100000:0.0:1767321516.764871:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985700608:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321516.764873:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af18e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985700480:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321516.765289:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321516.765295:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321516.892644:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321517.916071:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321518.251460:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321518.251554:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920677888 00000100:00100000:2.0:1767321518.251568:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920677888 00000100:00100000:2.0:1767321518.251713:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5869 00000100:00100000:2.0:1767321518.251725:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920677888:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321518.251814:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920677888:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 93us (373us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321518.251820:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5869 00000100:00100000:0.0:1767321518.252076:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321518.252191:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920677760 00000100:00100000:1.0:1767321518.252196:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920677760 00000100:00100000:1.0:1767321518.252216:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6765 00000100:00100000:1.0:1767321518.252220:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920677760:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321518.252243:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321518.252292:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920677760:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 71us (665us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321518.252299:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6765 00000100:00080000:3.0:1767321518.252830:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:2.0:1767321518.254505:0:7007:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920678016 offset 0 length 520: 4 00000100:00100000:1.0:1767321518.299360:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321493/real 1767321493] req@ffff8ce184a11880 x1853163985694208/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321518 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321518.299409:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce184a11880 x1853163985694208/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321518 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321518.299434:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321518.299466:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321518.299471:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a11880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985694208:0@lo:250:kworker.0 00000100:00080000:3.0:1767321518.300003:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321518.300010:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321518.300015:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321518.300020:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6741 00000100:00080000:3.0:1767321518.300026:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321518.300033:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321518.300076:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df8a80 x1853163985700736/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321518.300111:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321518.300115:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321518.300117:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321518.300121:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6762 00000100:00080000:3.0:1767321518.300124:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321518.300128:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321518.300143:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb800 x1853163985700864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321518.300151:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321518.300154:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321518.300155:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321518.300158:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6762 00000100:00080000:3.0:1767321518.300160:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321518.300187:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321518.300198:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985700992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321518.300249:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321518.300261:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321518.300263:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321518.300268:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6762 00000100:00080000:3.0:1767321518.300270:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321518.300274:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321518.300287:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb480 x1853163985701248/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321518.300323:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321518.300327:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321518.300329:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321518.300333:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6762 00000100:00080000:3.0:1767321518.300336:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321518.300339:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321518.300352:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9c00 x1853163985701504/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321518.300368:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321518.300387:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985701120:0@lo:400:kworker.0 00000100:00100000:0.0:1767321518.300476:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321518.300481:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985701376:0@lo:400:kworker.0 00000100:00100000:3.0:1767321518.300523:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321518.300559:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985701760:0@lo:400:kworker.0 00000100:00100000:1.0:1767321518.300593:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985701632:0@lo:400:kworker.0 00000100:00100000:0.0:1767321518.300595:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321518.300947:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985701120 00000100:00100000:2.0:1767321518.300957:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985701120 00000100:00100000:2.0:1767321518.301031:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985701376 00000100:00100000:2.0:1767321518.301035:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985701376 00000100:00100000:2.0:1767321518.301063:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5870 00000100:00100000:2.0:1767321518.301074:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985701120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321518.301152:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985701760 00000100:00100000:3.0:1767321518.301156:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985701760 00000100:00100000:3.0:1767321518.301216:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3785 00000100:00100000:3.0:1767321518.301224:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985701760:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321518.303982:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5871 00000100:00100000:0.0:1767321518.304011:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985701376:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321518.304415:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985701376:12345-0@lo:400:kworker.0 Request processed in 417us (3890us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321518.304424:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5871 00000100:00100000:0.0:1767321518.304506:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985700736:0@lo:250:kworker.0 00000400:00000100:0.0:1767321518.304547:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985700736 offset 0 length 520: 4 00000100:00100000:0.0:1767321518.304576:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985700864:0@lo:38:kworker.0 00000100:00100000:0.0:1767321518.304607:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321518.304626:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985700992:0@lo:38:kworker.0 00000100:00100000:0.0:1767321518.304651:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321518.304670:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985701248:0@lo:38:kworker.0 00000100:00100000:0.0:1767321518.304693:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321518.304709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985701504:0@lo:38:kworker.0 00000100:00100000:0.0:1767321518.304734:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321518.304768:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985700864 00000100:00100000:0.0:1767321518.304788:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985700864 00000100:00100000:0.0:1767321518.304819:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985701248 00000100:00100000:0.0:1767321518.304822:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985701248 00000100:00100000:0.0:1767321518.304830:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985701504 00000100:00100000:0.0:1767321518.304832:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985701504 00000100:00100000:0.0:1767321518.304840:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6766 00000100:00100000:0.0:1767321518.304845:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985700864:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321518.304867:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321518.304916:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985700864:12345-0@lo:38:kworker.0 Request processed in 69us (309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321518.304923:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6766 00000100:00100000:0.0:1767321518.304938:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6767 00000100:00100000:0.0:1767321518.304942:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985701248:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321518.304951:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321518.304973:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985701248:12345-0@lo:38:kworker.0 Request processed in 29us (280us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321518.304980:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6767 00000100:00100000:0.0:1767321518.304991:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6768 00000100:00100000:0.0:1767321518.304995:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985701504:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321518.305004:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321518.305027:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985701504:12345-0@lo:38:kworker.0 Request processed in 30us (293us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321518.305031:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6768 00000100:00080000:0.0:1767321518.305071:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321518.305076:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321518.305079:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985700864:0@lo:38:kworker.0 00000100:00080000:0.0:1767321518.305084:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321518.305087:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321518.305089:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985701248:0@lo:38:kworker.0 00000100:00080000:0.0:1767321518.305093:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321518.305095:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321518.305097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985701504:0@lo:38:kworker.0 00000100:00100000:0.0:1767321518.305147:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985700992 00000100:00100000:0.0:1767321518.305150:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985700992 00000100:00100000:0.0:1767321518.305186:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2182 00000100:00100000:0.0:1767321518.305189:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985700992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321518.305196:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321518.305216:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985700992:12345-0@lo:38:kworker.0 Request processed in 27us (566us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321518.305220:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2182 00000100:00080000:0.0:1767321518.305237:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321518.305239:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321518.305241:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985700992:0@lo:38:kworker.0 00000100:00100000:0.0:1767321518.305275:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985701376:0@lo:400:kworker.0 00000100:00080000:0.0:1767321518.305296:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321518.305301:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321518.306484:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985701120:12345-0@lo:400:kworker.0 Request processed in 5405us (6003us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321518.306494:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5870 00000100:00100000:2.0:1767321518.306551:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985701120:0@lo:400:kworker.0 00000100:00100000:3.0:1767321518.306758:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985701760:12345-0@lo:400:kworker.0 Request processed in 5532us (6160us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321518.306768:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3785 00000100:00100000:1.0:1767321518.306836:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321518.306911:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985701632 00000100:00100000:3.0:1767321518.306916:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985701632 00000100:00100000:3.0:1767321518.306962:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3786 00000100:00100000:3.0:1767321518.306968:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985701632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321518.307010:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985701632:12345-0@lo:400:kworker.0 Request processed in 43us (175us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321518.307015:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3786 00000100:00100000:1.0:1767321518.307120:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985701632:0@lo:400:kworker.0 00000100:00100000:2.0:1767321518.307273:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985701760:0@lo:400:kworker.0 00000100:00080000:0.0:1767321518.308206:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321518.940074:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321519.963926:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321520.989242:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321521.884647:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985701888:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321521.884759:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321521.885412:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985701888 00000100:00100000:0.0:1767321521.885424:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985701888 00000100:00100000:0.0:1767321521.885501:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3018 00000100:00100000:0.0:1767321521.885513:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985701888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321521.885550:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321521.885651:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985702016:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321521.885689:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321521.885753:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702016 00000100:00100000:2.0:1767321521.885758:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702016 00000100:00100000:2.0:1767321521.885789:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3019 00000100:00100000:2.0:1767321521.885797:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985702016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321521.885816:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321521.885851:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985702016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (162us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321521.885858:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3019 00000100:00100000:2.0:1767321521.885911:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985702016:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321521.885955:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321521.885960:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321521.888224:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985701888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2699us (3450us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321521.888243:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3018 00000100:00100000:1.0:1767321521.897969:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985701888:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321522.011554:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321523.035505:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321523.370065:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321523.370399:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920678144 00000100:00100000:0.0:1767321523.370413:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920678144 00000100:00100000:0.0:1767321523.370482:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6769 00000100:00100000:0.0:1767321523.370493:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920678144:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321523.370529:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321523.370616:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920678144:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 123us (566us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321523.370625:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6769 00000100:00100000:3.0:1767321523.370993:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321523.371132:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920678272 00000100:00100000:1.0:1767321523.371138:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920678272 00000100:00100000:1.0:1767321523.371252:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5872 00000100:00100000:1.0:1767321523.371261:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920678272:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321523.371319:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920678272:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 59us (327us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321523.371326:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5872 00000100:00080000:2.0:1767321523.371469:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321523.419830:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321523.419839:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321523.419842:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321523.419845:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321523.419849:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321523.419857:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6766 00000100:00080000:3.0:1767321523.419862:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321523.419868:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321523.420298:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfbb80 x1853163985702144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321523.420327:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321523.420330:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321523.420332:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321523.420335:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6766 00000100:00080000:3.0:1767321523.420337:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321523.420340:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321523.420352:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfad80 x1853163985702272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321523.420818:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985702400:0@lo:400:kworker.0 00000100:00100000:2.0:1767321523.420891:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321523.421025:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985702144:0@lo:38:kworker.0 00000100:00100000:1.0:1767321523.421059:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321523.421087:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985702272:0@lo:38:kworker.0 00000100:00100000:1.0:1767321523.421112:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321523.421154:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702272 00000100:00100000:1.0:1767321523.421185:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702272 00000100:00100000:1.0:1767321523.421239:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2183 00000100:00100000:1.0:1767321523.421247:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985702272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321523.421271:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321523.421310:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985702272:12345-0@lo:38:kworker.0 Request processed in 64us (197us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321523.421325:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2183 00000100:00100000:1.0:1767321523.421348:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702400 00000100:00100000:1.0:1767321523.421351:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702400 00000100:00100000:1.0:1767321523.421385:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702144 00000100:00100000:1.0:1767321523.421387:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702144 00000100:00100000:1.0:1767321523.421397:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5873 00000100:00100000:1.0:1767321523.421401:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985702400:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321523.421465:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321523.421468:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321523.421471:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321523.421475:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6766 00000100:00080000:3.0:1767321523.421477:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321523.421480:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321523.421494:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9180 x1853163985702528/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321523.421514:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321523.421516:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321523.421517:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321523.421520:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6766 00000100:00080000:3.0:1767321523.421521:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321523.421523:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321523.421529:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985702784/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321523.422382:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6770 00000100:00100000:0.0:1767321523.422391:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985702144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321523.422408:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321523.422457:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985702144:12345-0@lo:38:kworker.0 Request processed in 65us (1397us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321523.422464:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6770 00000100:00080000:0.0:1767321523.422485:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321523.422501:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985702528:0@lo:38:kworker.0 00000100:00100000:0.0:1767321523.422532:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321523.422566:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985702784:0@lo:38:kworker.0 00000100:00100000:0.0:1767321523.422593:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321523.422629:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321523.422634:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321523.422639:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985702144:0@lo:38:kworker.0 00000100:00080000:0.0:1767321523.422648:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321523.422652:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321523.422655:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985702272:0@lo:38:kworker.0 00000100:00100000:0.0:1767321523.422685:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702528 00000100:00100000:0.0:1767321523.422690:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702528 00000100:00100000:0.0:1767321523.422711:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702784 00000100:00100000:0.0:1767321523.422713:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702784 00000100:00100000:0.0:1767321523.422723:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6771 00000100:00100000:0.0:1767321523.422727:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985702528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321523.422737:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321523.422763:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985702528:12345-0@lo:38:kworker.0 Request processed in 34us (230us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321523.422768:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6771 00000100:00100000:0.0:1767321523.422781:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6772 00000100:00100000:0.0:1767321523.422785:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985702784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321523.422793:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321523.422813:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985702784:12345-0@lo:38:kworker.0 Request processed in 26us (219us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321523.422819:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6772 00000100:00080000:0.0:1767321523.422845:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321523.422849:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321523.422852:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985702528:0@lo:38:kworker.0 00000100:00080000:0.0:1767321523.422859:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321523.422862:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321523.422865:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985702784:0@lo:38:kworker.0 00000100:00100000:0.0:1767321523.422913:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985702912:0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.422945:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321523.422968:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321523.422975:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985702656:0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.423011:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321523.423035:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702656 00000100:00100000:0.0:1767321523.423039:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702656 00000100:00100000:0.0:1767321523.423075:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5874 00000100:00100000:0.0:1767321523.423081:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985702656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.423119:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985702656:12345-0@lo:400:kworker.0 Request processed in 37us (106us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321523.423125:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5874 00000100:00080000:0.0:1767321523.423141:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321523.423191:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985702656:0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.423229:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985702912 00000100:00100000:0.0:1767321523.423233:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985702912 00000100:00100000:0.0:1767321523.423273:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3787 00000100:00100000:0.0:1767321523.423278:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985702912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.423313:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985702912:12345-0@lo:400:kworker.0 Request processed in 35us (368us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321523.423319:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3787 00000100:00080000:0.0:1767321523.423333:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321523.423347:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985702912:0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.423381:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985703040:0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.423406:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321523.423433:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703040 00000100:00100000:0.0:1767321523.423437:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703040 00000100:00100000:0.0:1767321523.423461:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3788 00000100:00100000:0.0:1767321523.423466:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985703040:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321523.423500:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985703040:12345-0@lo:400:kworker.0 Request processed in 33us (93us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321523.423509:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3788 00000100:00080000:0.0:1767321523.423524:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321523.423538:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985703040:0@lo:400:kworker.0 00000100:00100000:1.0:1767321523.423647:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985702400:12345-0@lo:400:kworker.0 Request processed in 2243us (2754us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321523.423656:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5873 00000100:00100000:2.0:1767321523.423738:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985702400:0@lo:400:kworker.0 00010000:00010000:0.0:1767321524.059517:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321525.083520:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321526.107488:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321527.003586:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985703168:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321527.003683:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321527.004070:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703168 00000100:00100000:0.0:1767321527.004082:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703168 00000100:00100000:0.0:1767321527.004210:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3020 00000100:00100000:0.0:1767321527.004222:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985703168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321527.004262:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321527.004325:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985703168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (641us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321527.004333:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3020 00000100:00100000:1.0:1767321527.004466:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321527.004480:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985703296:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321527.004503:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321527.004518:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321527.004546:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985703168:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321527.004608:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703296 00000100:00100000:3.0:1767321527.004613:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703296 00000100:00100000:3.0:1767321527.004658:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3021 00000100:00100000:3.0:1767321527.004666:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985703296:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321527.004670:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:3.0:1767321527.004688:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321527.004740:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985703296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (221us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321527.004746:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3021 00000100:00100000:0.0:1767321527.005025:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985703296:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321527.131679:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321528.156510:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321528.490279:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321528.490333:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321528.490923:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920678400 00000100:00100000:0.0:1767321528.490937:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920678400 00000100:00100000:0.0:1767321528.491046:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920678528 00000100:00100000:0.0:1767321528.491050:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920678528 00000100:00100000:0.0:1767321528.491099:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5875 00000100:00100000:0.0:1767321528.491107:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920678528:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:3.0:1767321528.491223:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321528.491249:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920678528:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 142us (915us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321528.491254:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6773 00000100:00100000:0.0:1767321528.491260:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5875 00000100:00100000:3.0:1767321528.491262:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920678400:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321528.491283:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321528.491323:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920678400:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 60us (1056us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321528.491330:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6773 00000100:00080000:3.0:1767321528.539318:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321528.539328:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321528.539332:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321528.539336:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321528.539340:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321528.539350:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6771 00000100:00080000:3.0:1767321528.539355:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321528.539363:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321528.539406:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfb480 x1853163985703424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321528.539431:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321528.539435:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321528.539437:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321528.539440:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6771 00000100:00080000:3.0:1767321528.539443:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321528.539446:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321528.539460:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9c00 x1853163985703552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321528.539499:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321528.539504:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321528.539505:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321528.539509:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6771 00000100:00080000:3.0:1767321528.539512:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321528.539515:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321528.539525:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985703808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321528.539553:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321528.539556:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321528.539558:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321528.539561:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6771 00000100:00080000:3.0:1767321528.539563:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321528.539566:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321528.539576:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4dfad80 x1853163985704064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321528.539721:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985704192:0@lo:400:kworker.0 00000100:00100000:0.0:1767321528.539786:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985703424:0@lo:38:kworker.0 00000100:00100000:3.0:1767321528.539787:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321528.539819:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321528.539847:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985703552:0@lo:38:kworker.0 00000100:00100000:3.0:1767321528.539850:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321528.539856:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985703936:0@lo:400:kworker.0 00000100:00100000:0.0:1767321528.539869:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321528.539873:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321528.539892:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985703808:0@lo:38:kworker.0 00000100:00100000:3.0:1767321528.539905:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704192 00000100:00100000:0.0:1767321528.539909:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321528.539912:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704192 00000100:00100000:0.0:1767321528.539924:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985704064:0@lo:38:kworker.0 00000100:00100000:0.0:1767321528.539940:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321528.539958:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3789 00000100:00100000:0.0:1767321528.539966:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703552 00000100:00100000:3.0:1767321528.539967:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985704192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321528.539969:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703552 00000100:00100000:0.0:1767321528.539989:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2184 00000100:00100000:0.0:1767321528.539994:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985703552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321528.540011:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321528.540013:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985704192:12345-0@lo:400:kworker.0 Request processed in 47us (226us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321528.540020:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3789 00000100:00100000:0.0:1767321528.540038:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985703552:12345-0@lo:38:kworker.0 Request processed in 42us (168us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321528.540043:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2184 00000100:00100000:0.0:1767321528.540063:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703424 00000100:00100000:0.0:1767321528.540066:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703424 00000100:00100000:3.0:1767321528.540069:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985704192:0@lo:400:kworker.0 00000100:00100000:0.0:1767321528.540082:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703936 00000100:00100000:3.0:1767321528.540089:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703808 00000100:00100000:3.0:1767321528.540092:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703808 00000100:00100000:0.0:1767321528.540098:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703936 00000100:00100000:3.0:1767321528.540107:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704064 00000100:00100000:3.0:1767321528.540110:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704064 00000100:00100000:3.0:1767321528.540120:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5876 00000100:00100000:0.0:1767321528.540135:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6774 00000100:00100000:3.0:1767321528.540138:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985703936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321528.540140:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985703424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321528.540150:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321528.540200:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985703424:12345-0@lo:38:kworker.0 Request processed in 59us (380us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321528.540205:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6774 00000100:00100000:0.0:1767321528.540216:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6775 00000100:00100000:0.0:1767321528.540220:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985703808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321528.540228:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321528.540249:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985703808:12345-0@lo:38:kworker.0 Request processed in 27us (338us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321528.540254:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6775 00000100:00100000:0.0:1767321528.540265:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6776 00000100:00100000:0.0:1767321528.540269:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985704064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321528.540277:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321528.540294:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985704064:12345-0@lo:38:kworker.0 Request processed in 24us (354us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321528.540300:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6776 00000100:00080000:0.0:1767321528.540321:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321528.540339:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321528.540343:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321528.540347:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985703424:0@lo:38:kworker.0 00000100:00080000:0.0:1767321528.540355:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321528.540358:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321528.540361:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985703552:0@lo:38:kworker.0 00000100:00080000:0.0:1767321528.540368:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321528.540371:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321528.540374:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985703808:0@lo:38:kworker.0 00000100:00080000:0.0:1767321528.540381:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321528.540384:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321528.540386:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4dfad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985704064:0@lo:38:kworker.0 00000100:00100000:3.0:1767321528.541023:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985703936:12345-0@lo:400:kworker.0 Request processed in 883us (1146us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321528.541031:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5876 00000100:00100000:0.0:1767321528.541063:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985703936:0@lo:400:kworker.0 00000100:00100000:2.0:1767321528.541230:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985704320:0@lo:400:kworker.0 00000100:00100000:2.0:1767321528.541256:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321528.541280:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321528.541284:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985703680:0@lo:400:kworker.0 00000100:00100000:2.0:1767321528.541298:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321528.541333:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985703680 00000100:00100000:0.0:1767321528.541340:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985703680 00000100:00100000:3.0:1767321528.541422:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704320 00000100:00100000:3.0:1767321528.541426:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704320 00000100:00100000:0.0:1767321528.541439:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5877 00000100:00100000:0.0:1767321528.541446:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985703680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321528.541449:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3790 00000100:00100000:3.0:1767321528.541454:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985704320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321528.541481:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985703680:12345-0@lo:400:kworker.0 Request processed in 36us (181us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321528.541487:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5877 00000100:00080000:2.0:1767321528.541665:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321528.543500:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985704320:12345-0@lo:400:kworker.0 Request processed in 2037us (2233us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321528.543513:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3790 00000100:00100000:2.0:1767321528.543572:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985703680:0@lo:400:kworker.0 00000100:00100000:2.0:1767321528.543589:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985704320:0@lo:400:kworker.0 00010000:00010000:2.0:1767321529.180217:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321530.204432:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321531.227467:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321532.124490:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985704448:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321532.124596:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321532.124719:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704448 00000100:00100000:1.0:1767321532.124731:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704448 00000100:00080000:3.0:1767321532.124803:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321532.124814:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3022 00000100:00100000:1.0:1767321532.124824:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985704448:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321532.124861:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321532.124913:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985704448:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321532.124921:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3022 00000100:00100000:2.0:1767321532.124954:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985704448:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321532.125863:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985704576:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321532.125923:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321532.131669:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704576 00000100:00100000:2.0:1767321532.131703:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704576 00000100:00100000:2.0:1767321532.131908:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3023 00000100:00100000:2.0:1767321532.131923:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985704576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321532.131971:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321532.132114:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985704576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 194us (6189us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321532.132124:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3023 00000100:00100000:1.0:1767321532.132330:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985704576:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321532.137961:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321532.251573:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321533.275810:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321533.610273:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321533.610335:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321533.610417:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920678656 00000100:00100000:1.0:1767321533.610449:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920678656 00000100:00100000:1.0:1767321533.610535:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920678784 00000100:00100000:1.0:1767321533.610539:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920678784 00000100:00100000:1.0:1767321533.610615:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5878 00000100:00100000:1.0:1767321533.610630:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920678784:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321533.610714:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6777 00000100:00100000:2.0:1767321533.610725:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920678656:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:1.0:1767321533.610750:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920678784:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 123us (412us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321533.610759:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5878 00010000:00020000:2.0:1767321533.610767:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321533.610806:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920678656:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 82us (545us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321533.610814:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6777 00000100:00080000:3.0:1767321533.611068:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321533.659376:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321533.659393:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321533.659397:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321533.659401:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321533.659406:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321533.659417:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6776 00000100:00080000:0.0:1767321533.659423:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321533.659432:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321533.659586:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e0a80 x1853163985704704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321533.659628:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321533.659632:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321533.659633:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321533.659637:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6776 00000100:00080000:0.0:1767321533.659640:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321533.659644:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321533.659660:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e0000 x1853163985704832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321533.659715:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321533.659719:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321533.659721:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321533.659725:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6776 00000100:00080000:0.0:1767321533.659728:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321533.659732:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321533.659744:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e1180 x1853163985705088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321533.659779:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321533.659782:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321533.659785:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321533.659792:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6776 00000100:00080000:0.0:1767321533.659795:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767321533.659796:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985704960:0@lo:400:kworker.0 00000100:00080000:0.0:1767321533.659802:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321533.659821:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bead80 x1853163985705344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321533.659903:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985705216:0@lo:400:kworker.0 00000100:00100000:2.0:1767321533.659916:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985705472:0@lo:400:kworker.0 00000100:00100000:3.0:1767321533.659917:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321533.659980:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321533.659989:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321533.660050:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321533.660071:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985705600:0@lo:400:kworker.0 00000100:00100000:3.0:1767321533.660106:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321533.660156:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704960 00000100:00100000:3.0:1767321533.660188:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704960 00000100:00100000:3.0:1767321533.660241:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705216 00000100:00100000:3.0:1767321533.660245:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705216 00000100:00100000:3.0:1767321533.660267:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5879 00000100:00100000:3.0:1767321533.660276:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985704960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321533.660328:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985704960:12345-0@lo:400:kworker.0 Request processed in 53us (412us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321533.660335:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5879 00000100:00100000:3.0:1767321533.660351:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5880 00000100:00100000:3.0:1767321533.660358:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985705216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321533.660384:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985705216:12345-0@lo:400:kworker.0 Request processed in 27us (408us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321533.660390:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5880 00000100:00100000:3.0:1767321533.660410:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705472 00000100:00100000:3.0:1767321533.660413:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705472 00000100:00100000:3.0:1767321533.660438:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705600 00000100:00100000:3.0:1767321533.660441:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705600 00000100:00100000:3.0:1767321533.660459:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3791 00000100:00100000:3.0:1767321533.660465:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985705472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321533.660495:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985705472:12345-0@lo:400:kworker.0 Request processed in 30us (507us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321533.660500:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3791 00000100:00100000:3.0:1767321533.660510:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3792 00000100:00100000:3.0:1767321533.660515:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985705600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321533.660537:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985705600:12345-0@lo:400:kworker.0 Request processed in 22us (431us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321533.660544:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3792 00000100:00080000:3.0:1767321533.660565:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321533.660595:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985705600:0@lo:400:kworker.0 00000100:00100000:3.0:1767321533.660606:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985704960:0@lo:400:kworker.0 00000100:00100000:1.0:1767321533.660660:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985704704:0@lo:38:kworker.0 00000100:00100000:1.0:1767321533.660717:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321533.660757:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985704832:0@lo:38:kworker.0 00000100:00100000:1.0:1767321533.660784:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321533.660810:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985705088:0@lo:38:kworker.0 00000100:00100000:1.0:1767321533.660832:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321533.660848:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985705344:0@lo:38:kworker.0 00000100:00100000:1.0:1767321533.660870:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321533.660912:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704832 00000100:00100000:0.0:1767321533.661601:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985705216:0@lo:400:kworker.0 00000100:00100000:0.0:1767321533.661657:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985704704 00000100:00100000:0.0:1767321533.661664:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704704 00000100:00100000:0.0:1767321533.661706:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705088 00000100:00100000:0.0:1767321533.661709:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705088 00000100:00100000:0.0:1767321533.661724:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705344 00000100:00100000:0.0:1767321533.661727:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705344 00000100:00100000:0.0:1767321533.661745:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6778 00000100:00100000:0.0:1767321533.661752:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985704704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321533.661779:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321533.661839:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985704704:12345-0@lo:38:kworker.0 Request processed in 86us (1122us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321533.661847:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6778 00000100:00100000:0.0:1767321533.661865:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6779 00000100:00100000:0.0:1767321533.661870:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985705088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321533.661879:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321533.661903:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985705088:12345-0@lo:38:kworker.0 Request processed in 32us (1070us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321533.661909:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6779 00000100:00100000:0.0:1767321533.661920:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6780 00000100:00100000:0.0:1767321533.661924:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985705344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321533.661933:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321533.661957:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985705344:12345-0@lo:38:kworker.0 Request processed in 31us (1086us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321533.661962:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6780 00000100:00080000:0.0:1767321533.662051:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321533.662058:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321533.662069:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985704704:0@lo:38:kworker.0 00000100:00080000:0.0:1767321533.662081:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321533.662086:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321533.662089:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985705088:0@lo:38:kworker.0 00000100:00080000:0.0:1767321533.662098:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321533.662101:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321533.662104:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985705344:0@lo:38:kworker.0 00000100:00100000:1.0:1767321533.662529:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985704832 00000100:00100000:1.0:1767321533.662575:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2185 00000100:00100000:1.0:1767321533.662580:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985704832:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321533.662594:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321533.662650:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985704832:12345-0@lo:38:kworker.0 Request processed in 68us (1865us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321533.662657:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2185 00000100:00080000:1.0:1767321533.662691:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321533.662696:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321533.662700:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985704832:0@lo:38:kworker.0 00000100:00100000:2.0:1767321533.663998:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985705472:0@lo:400:kworker.0 00010000:00010000:1.0:1767321534.299420:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321535.323415:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321536.347520:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321537.243426:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321537.243468:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985705728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321537.243582:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321537.243857:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705728 00000100:00100000:2.0:1767321537.243869:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705728 00000100:00100000:2.0:1767321537.243960:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3024 00000100:00100000:2.0:1767321537.243984:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985705728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321537.244027:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321537.244094:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985705728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (512us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321537.244103:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3024 00000100:00100000:1.0:1767321537.244209:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985705728:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321537.244311:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321537.244515:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985705856:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321537.244563:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321537.244644:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705856 00000100:00100000:1.0:1767321537.244652:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705856 00000100:00100000:1.0:1767321537.244723:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3025 00000100:00100000:1.0:1767321537.244732:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985705856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321537.244771:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321537.244845:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985705856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 114us (281us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321537.244852:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3025 00000100:00100000:1.0:1767321537.244892:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985705856:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321537.245062:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321537.371499:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321538.396110:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321538.731077:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321538.731231:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321538.731371:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920679040 00000100:00100000:1.0:1767321538.731385:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920679040 00000100:00100000:0.0:1767321538.731434:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920678912 00000100:00100000:0.0:1767321538.731442:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920678912 00000100:00100000:1.0:1767321538.731488:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5881 00000100:00100000:0.0:1767321538.731498:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6781 00000100:00100000:1.0:1767321538.731504:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920679040:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321538.731509:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920678912:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321538.731543:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321538.731620:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920679040:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 120us (557us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321538.731630:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5881 00000100:00080000:3.0:1767321538.731818:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321538.733855:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920678912:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 2330us (2606us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321538.733872:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6781 00000100:00080000:3.0:1767321538.779408:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321538.779421:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321538.779424:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321538.779430:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321538.779435:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321538.779447:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6781 00000100:00080000:3.0:1767321538.779453:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321538.779461:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321538.779516:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8700 x1853163985705984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321538.779543:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321538.779548:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321538.779550:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321538.779554:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6781 00000100:00080000:3.0:1767321538.779558:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321538.779562:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321538.779577:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bda680 x1853163985706112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321538.779666:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321538.779761:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985705984:0@lo:38:kworker.0 00000100:00100000:1.0:1767321538.779850:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321538.779969:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985706112:0@lo:38:kworker.0 00000100:00100000:1.0:1767321538.780000:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321538.780060:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985706112 00000100:00100000:1.0:1767321538.780071:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985706112 00000100:00100000:1.0:1767321538.780115:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2186 00000100:00100000:1.0:1767321538.780123:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985706112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321538.780149:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321538.780225:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985706112:12345-0@lo:38:kworker.0 Request processed in 101us (222us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321538.780233:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2186 00000100:00080000:1.0:1767321538.780284:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321538.780290:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321538.780294:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985706112:0@lo:38:kworker.0 00000100:00100000:1.0:1767321538.780343:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985705984 00000100:00100000:1.0:1767321538.780347:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985705984 00000100:00100000:1.0:1767321538.780374:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6782 00000100:00100000:1.0:1767321538.780379:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985705984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321538.780389:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321538.780419:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985705984:12345-0@lo:38:kworker.0 Request processed in 39us (569us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321538.780424:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6782 00000100:00080000:1.0:1767321538.780447:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321538.780451:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321538.780454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985705984:0@lo:38:kworker.0 00000100:00080000:3.0:1767321538.780532:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321538.780537:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321538.780539:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321538.780544:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6781 00000100:00080000:3.0:1767321538.780547:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321538.780552:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321538.780571:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb800 x1853163985706368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321538.780615:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321538.780619:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321538.780621:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321538.780625:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6781 00000100:00080000:3.0:1767321538.780628:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321538.780632:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321538.780644:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdb100 x1853163985706624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321538.780710:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985706368:0@lo:38:kworker.0 00000100:00100000:1.0:1767321538.780743:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321538.780764:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985706624:0@lo:38:kworker.0 00000100:00100000:1.0:1767321538.780789:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321538.780821:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985706368 00000100:00100000:1.0:1767321538.780825:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985706368 00000100:00100000:1.0:1767321538.780843:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985706624 00000100:00100000:1.0:1767321538.780846:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985706624 00000100:00100000:1.0:1767321538.780864:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6783 00000100:00100000:1.0:1767321538.780869:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985706368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321538.780881:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321538.780914:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985706368:12345-0@lo:38:kworker.0 Request processed in 44us (171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321538.780921:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6783 00000100:00100000:1.0:1767321538.780933:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6784 00000100:00100000:1.0:1767321538.780950:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985706624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321538.780960:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321538.780983:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985706624:12345-0@lo:38:kworker.0 Request processed in 32us (194us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321538.780989:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6784 00000100:00080000:1.0:1767321538.781012:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321538.781016:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321538.781019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985706368:0@lo:38:kworker.0 00000100:00080000:1.0:1767321538.781028:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321538.781031:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321538.781033:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985706624:0@lo:38:kworker.0 00000100:00100000:1.0:1767321538.781096:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985706496:0@lo:400:kworker.0 00000100:00100000:1.0:1767321538.781121:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321538.781143:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321538.781148:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985706752:0@lo:400:kworker.0 00000100:00100000:1.0:1767321538.781196:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321538.781225:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985706496 00000100:00100000:1.0:1767321538.781229:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985706496 00000100:00100000:1.0:1767321538.781276:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5882 00000100:00100000:1.0:1767321538.781283:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985706496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321538.781328:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985706496:12345-0@lo:400:kworker.0 Request processed in 45us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321538.781334:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5882 00000100:00100000:1.0:1767321538.781362:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985706496:0@lo:400:kworker.0 00000100:00100000:1.0:1767321538.781400:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985706752 00000100:00100000:1.0:1767321538.781404:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985706752 00000100:00100000:1.0:1767321538.781436:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3793 00000100:00100000:1.0:1767321538.781443:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985706752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321538.781480:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985706752:12345-0@lo:400:kworker.0 Request processed in 37us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321538.781486:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3793 00000100:00100000:1.0:1767321538.781515:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985706752:0@lo:400:kworker.0 00000100:00100000:2.0:1767321538.781610:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321538.781624:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985706880:0@lo:400:kworker.0 00000100:00100000:2.0:1767321538.781659:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321538.781707:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321538.781880:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985706880 00000100:00100000:1.0:1767321538.781886:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985706880 00000100:00100000:1.0:1767321538.781920:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3794 00000100:00100000:1.0:1767321538.781927:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985706880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321538.781977:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985706880:12345-0@lo:400:kworker.0 Request processed in 50us (318us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321538.781985:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3794 00000100:00100000:2.0:1767321538.782134:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985706880:0@lo:400:kworker.0 00000100:00100000:0.0:1767321538.782232:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985706240:0@lo:400:kworker.0 00000100:00100000:0.0:1767321538.782267:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321538.786208:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985706240 00000100:00100000:2.0:1767321538.786232:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985706240 00000100:00100000:2.0:1767321538.788379:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5883 00000100:00100000:2.0:1767321538.788397:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985706240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321538.788527:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985706240:12345-0@lo:400:kworker.0 Request processed in 134us (6258us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321538.788535:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5883 00000100:00100000:0.0:1767321538.789820:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985706240:0@lo:400:kworker.0 00000100:00080000:3.0:1767321538.792134:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321539.421319:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321540.443666:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321541.467971:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321542.363624:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321542.363657:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985707008:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321542.363754:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321542.363928:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985707136:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321542.364355:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321542.364772:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707008 00000100:00100000:3.0:1767321542.364782:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707008 00000100:00100000:3.0:1767321542.364850:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707136 00000100:00100000:3.0:1767321542.364853:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707136 00000100:00080000:2.0:1767321542.364968:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321542.364972:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321542.365018:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3026 00000100:00100000:0.0:1767321542.365030:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985707008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321542.365070:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321542.365137:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985707008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 110us (1383us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321542.365144:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3026 00000100:00100000:2.0:1767321542.365241:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985707008:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321542.365374:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3027 00000100:00100000:3.0:1767321542.365383:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985707136:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321542.365511:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321542.365555:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985707136:12345-0@lo:13:osp-pre-0-1.0 Request processed in 174us (1205us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321542.365562:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3027 00000100:00100000:2.0:1767321542.365627:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985707136:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767321542.491395:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321543.515627:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767321543.850585:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920679168 offset 0 length 520: 4 00000100:00100000:3.0:1767321543.850632:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321543.850737:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321543.857382:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920679296 00000100:00100000:1.0:1767321543.857402:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920679296 00000100:00100000:1.0:1767321543.857464:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920679424 00000100:00100000:1.0:1767321543.857468:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920679424 00000100:00100000:1.0:1767321543.857531:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5884 00000100:00080000:0.0:1767321543.857540:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321543.857543:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920679424:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321543.857580:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6785 00000100:00100000:0.0:1767321543.857588:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920679296:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321543.857613:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321543.857641:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920679424:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 100us (6902us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321543.857650:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2512:x1853163920679296:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 60us (7017us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321543.857652:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5884 00000100:00100000:0.0:1767321543.857663:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6785 00000100:00080000:0.0:1767321543.899782:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321543.899793:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321543.899796:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321543.899800:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321543.899805:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321543.899813:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6787 00000100:00080000:0.0:1767321543.899818:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321543.899825:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321543.899876:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb100 x1853163985707264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321543.899900:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321543.899904:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321543.899906:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321543.899910:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6787 00000100:00080000:0.0:1767321543.899912:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321543.899915:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321543.899926:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8000 x1853163985707392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321543.899993:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985707520:0@lo:400:kworker.0 00000100:00100000:1.0:1767321543.900058:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321543.900176:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707520 00000100:00100000:3.0:1767321543.900184:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707520 00000100:00100000:2.0:1767321543.900186:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321518/real 1767321518] req@ffff8ce1a4df8a80 x1853163985700736/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321543 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321543.900206:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a4df8a80 x1853163985700736/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321543 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321543.900222:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985707264:0@lo:38:kworker.0 00000100:00100000:2.0:1767321543.900249:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321543.900250:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5885 00000100:00100000:3.0:1767321543.900258:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985707520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321543.900270:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985707392:0@lo:38:kworker.0 00000100:00100000:2.0:1767321543.900294:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321543.900316:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321543.900326:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321543.900329:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985700736:0@lo:250:kworker.0 00000100:00080000:0.0:1767321543.900336:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321543.900340:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321543.900342:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321543.900345:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6787 00000100:00080000:0.0:1767321543.900348:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321543.900351:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321543.900364:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707392 00000100:00100000:0.0:1767321543.900367:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8380 x1853163985707648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321543.900367:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707392 00000100:00100000:2.0:1767321543.900389:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2187 00000100:00080000:0.0:1767321543.900394:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767321543.900394:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985707392:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767321543.900397:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321543.900399:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321543.900402:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6787 00000100:00080000:0.0:1767321543.900404:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321543.900407:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00010000:00020000:2.0:1767321543.900415:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321543.900417:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9c00 x1853163985707904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321543.900446:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985707392:12345-0@lo:38:kworker.0 Request processed in 50us (150us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321543.900453:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2187 00000100:00100000:2.0:1767321543.900473:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985707776:0@lo:400:kworker.0 00000100:00100000:2.0:1767321543.900496:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321543.900517:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321543.900522:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985708032:0@lo:400:kworker.0 00000100:00100000:2.0:1767321543.900545:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321543.900570:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985708032 00000100:00100000:2.0:1767321543.900575:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985708032 00000100:00100000:2.0:1767321543.900604:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3795 00000100:00100000:2.0:1767321543.900609:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985708032:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321543.900644:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985708032:12345-0@lo:400:kworker.0 Request processed in 34us (97us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321543.900649:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3795 00000100:00100000:2.0:1767321543.900682:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985708032:0@lo:400:kworker.0 00000100:00100000:2.0:1767321543.900704:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707264 00000100:00100000:2.0:1767321543.900707:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707264 00000100:00100000:2.0:1767321543.900726:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707776 00000100:00100000:2.0:1767321543.900730:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707776 00000100:00100000:2.0:1767321543.900753:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5886 00000100:00100000:2.0:1767321543.900759:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985707776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321543.900790:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985707776:12345-0@lo:400:kworker.0 Request processed in 31us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321543.900796:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5886 00000100:00100000:2.0:1767321543.900807:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6786 00000100:00100000:2.0:1767321543.900812:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985707264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321543.900822:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321543.900860:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985707264:12345-0@lo:38:kworker.0 Request processed in 46us (610us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321543.900865:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6786 00000100:00100000:2.0:1767321543.900889:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985707776:0@lo:400:kworker.0 00000100:00080000:2.0:1767321543.900918:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321543.900947:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985707648:0@lo:38:kworker.0 00000100:00100000:2.0:1767321543.900969:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321543.900984:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985707904:0@lo:38:kworker.0 00000100:00100000:2.0:1767321543.901001:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321543.901019:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321543.901023:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321543.901026:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985707264:0@lo:38:kworker.0 00000100:00080000:2.0:1767321543.901034:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321543.901037:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321543.901040:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985707392:0@lo:38:kworker.0 00000100:00100000:2.0:1767321543.901066:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985708160:0@lo:400:kworker.0 00000100:00100000:2.0:1767321543.901088:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321543.901113:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707648 00000100:00100000:2.0:1767321543.901117:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707648 00000100:00100000:2.0:1767321543.901131:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985707904 00000100:00100000:2.0:1767321543.901134:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985707904 00000100:00100000:2.0:1767321543.901143:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6787 00000100:00100000:2.0:1767321543.901147:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985707648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321543.901156:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321543.901207:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985707648:12345-0@lo:38:kworker.0 Request processed in 58us (236us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321543.901212:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6787 00000100:00100000:2.0:1767321543.901224:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6788 00000100:00100000:2.0:1767321543.901228:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985707904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321543.901236:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321543.901241:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985707520:12345-0@lo:400:kworker.0 Request processed in 984us (1182us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321543.901247:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5885 00000100:00100000:2.0:1767321543.901254:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a60a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985707904:12345-0@lo:38:kworker.0 Request processed in 26us (253us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321543.901259:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6788 00000100:00100000:3.0:1767321543.901273:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985708160 00000100:00100000:3.0:1767321543.901277:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985708160 00000100:00080000:2.0:1767321543.901278:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321543.901281:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321543.901284:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985707648:0@lo:38:kworker.0 00000100:00080000:2.0:1767321543.901291:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321543.901293:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321543.901296:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985707904:0@lo:38:kworker.0 00000100:00100000:3.0:1767321543.901297:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3796 00000100:00100000:3.0:1767321543.901302:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985708160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321543.901333:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a63480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985708160:12345-0@lo:400:kworker.0 Request processed in 30us (244us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321543.901339:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3796 00000100:00100000:2.0:1767321543.901347:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985708160:0@lo:400:kworker.0 00000100:00100000:1.0:1767321543.901594:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985707520:0@lo:400:kworker.0 00000100:00080000:0.0:1767321543.901772:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321543.901777:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321543.901780:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321543.901785:0:55800:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6766 00000100:00080000:0.0:1767321543.901788:0:55800:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321543.901793:0:55800:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321543.901815:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bead80 x1853163985708288/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321543.901905:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985708288:0@lo:250:kworker.0 00000400:00000100:2.0:1767321543.901924:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985708288 offset 0 length 520: 4 00010000:00010000:2.0:1767321544.539883:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321545.564550:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321546.588149:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321547.483484:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321547.483497:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985708416:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321547.483510:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985708544:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321547.484606:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321547.487694:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985708544 00000100:00100000:2.0:1767321547.487706:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985708544 00000100:00100000:2.0:1767321547.487809:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3028 00000100:00100000:2.0:1767321547.487819:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985708544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321547.487857:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321547.488279:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985708544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 461us (3677us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321547.488288:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3028 00000100:00100000:3.0:1767321547.488377:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985708544:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321547.488435:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321547.490708:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321547.491008:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985708416 00000100:00100000:2.0:1767321547.491027:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985708416 00000100:00080000:0.0:1767321547.491161:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321547.491395:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3029 00000100:00100000:2.0:1767321547.491408:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985708416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321547.491441:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321547.491511:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985708416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 108us (811us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321547.491518:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3029 00000100:00100000:1.0:1767321547.492461:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985708416:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321547.611475:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321548.636199:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321548.972341:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321548.972425:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920679552 00000100:00100000:0.0:1767321548.972435:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920679552 00000100:00100000:0.0:1767321548.972493:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6789 00000100:00100000:0.0:1767321548.972500:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920679552:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321548.972524:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321548.972607:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a62680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920679552:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 105us (274us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321548.972614:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6789 00000100:00100000:2.0:1767321548.973250:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321548.976498:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920679680 00000100:00100000:3.0:1767321548.976513:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920679680 00000100:00100000:3.0:1767321548.976801:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5887 00000100:00100000:3.0:1767321548.976813:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920679680:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321548.976956:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920679680:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 146us (3707us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321548.976965:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5887 00000100:00080000:0.0:1767321548.980015:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321549.020195:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321549.020205:0:55800:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321549.020208:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321549.020212:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321549.020216:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321549.020223:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6792 00000100:00080000:3.0:1767321549.020228:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321549.020235:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321549.020273:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bda300 x1853163985708672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321549.020302:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321549.020312:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321549.020314:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321549.020318:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6792 00000100:00080000:3.0:1767321549.020320:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321549.020324:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321549.020336:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bda680 x1853163985708800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321549.020385:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321549.020391:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321549.020393:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321549.020396:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6792 00000100:00080000:3.0:1767321549.020399:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321549.020402:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321549.020413:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd8700 x1853163985709056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321549.020436:0:55800:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321549.020440:0:55800:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321549.020441:0:55800:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321549.020445:0:55800:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6792 00000100:00080000:3.0:1767321549.020447:0:55800:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321549.020450:0:55800:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321549.020461:0:55800:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9880 x1853163985709312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321549.020498:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985709440:0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.020564:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321549.020630:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321549.020631:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985709568:0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.020639:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985709184:0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.020678:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321549.020690:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321549.020720:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709440 00000100:00100000:1.0:1767321549.020724:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321549.020728:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709440 00000100:00100000:1.0:1767321549.020733:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985708928:0@lo:400:kworker.0 00000100:00100000:1.0:1767321549.020771:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321549.020792:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709568 00000100:00100000:3.0:1767321549.020797:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709568 00000100:00100000:1.0:1767321549.020807:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709184 00000100:00100000:1.0:1767321549.020813:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709184 00000100:00100000:3.0:1767321549.020825:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3797 00000100:00100000:3.0:1767321549.020837:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985709440:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321549.020848:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985708928 00000100:00100000:1.0:1767321549.020854:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985708928 00000100:00100000:1.0:1767321549.020876:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5888 00000100:00100000:1.0:1767321549.020884:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985709184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.020913:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985709440:12345-0@lo:400:kworker.0 Request processed in 78us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321549.020926:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985709184:12345-0@lo:400:kworker.0 Request processed in 42us (246us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321549.020934:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5888 00000100:00100000:1.0:1767321549.020950:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5889 00000100:00100000:1.0:1767321549.020956:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985708928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321549.020988:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985708928:12345-0@lo:400:kworker.0 Request processed in 33us (218us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321549.020994:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5889 00000100:00100000:1.0:1767321549.021028:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985708928:0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.021203:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3797 00000100:00100000:3.0:1767321549.021224:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3798 00000100:00100000:3.0:1767321549.021231:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985709568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.021275:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985709568:12345-0@lo:400:kworker.0 Request processed in 44us (587us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321549.021281:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3798 00000100:00100000:3.0:1767321549.021309:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985709568:0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.021332:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985709184:0@lo:400:kworker.0 00000100:00100000:3.0:1767321549.021339:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985709440:0@lo:400:kworker.0 00000100:00080000:0.0:1767321549.021699:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321549.021704:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321549.021706:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321549.021790:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985708672:0@lo:38:kworker.0 00000100:00100000:2.0:1767321549.021826:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321549.021865:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985708800:0@lo:38:kworker.0 00000100:00100000:2.0:1767321549.021888:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321549.021918:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985709056:0@lo:38:kworker.0 00000100:00100000:2.0:1767321549.021948:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321549.021971:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985709312:0@lo:38:kworker.0 00000100:00100000:2.0:1767321549.021989:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321549.022018:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985708672 00000100:00100000:2.0:1767321549.022024:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985708672 00000100:00100000:2.0:1767321549.022046:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709056 00000100:00100000:2.0:1767321549.022048:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709056 00000100:00100000:2.0:1767321549.022059:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709312 00000100:00100000:2.0:1767321549.022062:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709312 00000100:00100000:2.0:1767321549.022073:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6790 00000100:00100000:2.0:1767321549.022078:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985708672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321549.022095:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321549.022128:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985708672:12345-0@lo:38:kworker.0 Request processed in 48us (301us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321549.022135:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6790 00000100:00100000:2.0:1767321549.022148:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6791 00000100:00100000:2.0:1767321549.022152:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985709056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321549.022160:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321549.022217:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985709056:12345-0@lo:38:kworker.0 Request processed in 63us (268us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321549.022223:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6791 00000100:00100000:2.0:1767321549.022235:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6792 00000100:00100000:2.0:1767321549.022239:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985709312:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321549.022248:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321549.022271:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985709312:12345-0@lo:38:kworker.0 Request processed in 31us (281us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321549.022277:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6792 00000100:00080000:2.0:1767321549.022307:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321549.022311:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321549.022315:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985708672:0@lo:38:kworker.0 00000100:00080000:2.0:1767321549.022322:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321549.022325:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321549.022327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985709056:0@lo:38:kworker.0 00000100:00080000:2.0:1767321549.022333:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321549.022336:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321549.022338:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985709312:0@lo:38:kworker.0 00000100:00100000:1.0:1767321549.022926:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985708800 00000100:00100000:1.0:1767321549.022934:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985708800 00000100:00100000:1.0:1767321549.022971:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2188 00000100:00100000:1.0:1767321549.022977:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985708800:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321549.022989:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321549.023052:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985708800:12345-0@lo:38:kworker.0 Request processed in 74us (1162us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321549.023059:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2188 00000100:00080000:0.0:1767321549.023215:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321549.023221:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321549.023225:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985708800:0@lo:38:kworker.0 00010000:00010000:1.0:1767321549.660426:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321550.684742:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321551.707483:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321552.603750:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985709696:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321552.603838:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321552.604065:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709696 00000100:00100000:3.0:1767321552.604074:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709696 00000100:00100000:3.0:1767321552.604132:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3030 00000100:00100000:3.0:1767321552.604140:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985709696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321552.604186:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321552.604227:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985709696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (399us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321552.604234:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3030 00000100:00080000:1.0:1767321552.604467:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321552.604506:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985709696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321552.605624:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321552.605643:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985709824:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321552.605713:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321552.605815:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709824 00000100:00100000:0.0:1767321552.605824:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709824 00000100:00100000:0.0:1767321552.605898:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3031 00000100:00100000:0.0:1767321552.605908:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985709824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321552.605936:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321552.605978:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985709824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321552.605986:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3031 00000100:00080000:3.0:1767321552.606197:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321552.606232:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985709824:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321552.734526:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321553.756972:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321554.092844:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321554.092986:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321554.093208:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920679936 00000100:00100000:0.0:1767321554.093222:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920679936 00000100:00100000:2.0:1767321554.093285:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920679808 00000100:00100000:2.0:1767321554.093303:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920679808 00000100:00100000:0.0:1767321554.093332:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5890 00000100:00100000:0.0:1767321554.093359:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920679936:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321554.093468:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920679936:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 124us (639us total) trans 0 rc 0/0 00000100:00080000:3.0:1767321554.093884:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321554.095543:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6793 00000100:00100000:2.0:1767321554.095555:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920679808:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321554.095581:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321554.095682:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920679808:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 120us (2688us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321554.095689:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6793 00000100:00100000:0.0:1767321554.099644:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5890 00000100:00080000:2.0:1767321554.139709:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321554.139721:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321554.139724:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321554.139730:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321554.139734:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321554.139742:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6797 00000100:00080000:2.0:1767321554.139747:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321554.139756:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321554.139805:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407b80 x1853163985709952/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321554.139826:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321554.139829:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321554.139831:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321554.139846:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6797 00000100:00100000:0.0:1767321554.139848:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985709952:0@lo:38:kworker.0 00000100:00080000:2.0:1767321554.139850:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321554.139860:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321554.139881:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404000 x1853163985710080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321554.139942:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321554.139946:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321554.139949:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321554.139954:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6797 00000100:00080000:2.0:1767321554.139956:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321554.139960:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321554.139978:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5404380 x1853163985710336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321554.139987:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321554.140027:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321554.140032:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321554.140035:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321554.140040:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6797 00000100:00080000:2.0:1767321554.140043:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321554.140046:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321554.140062:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407100 x1853163985710592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321554.140104:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985710080:0@lo:38:kworker.0 00000100:00100000:2.0:1767321554.140137:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985710464:0@lo:400:kworker.0 00000100:00100000:0.0:1767321554.140148:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321554.140234:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985710336:0@lo:38:kworker.0 00000100:00100000:2.0:1767321554.140250:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321554.140276:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321554.140308:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321554.140319:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985710592:0@lo:38:kworker.0 00000100:00100000:2.0:1767321554.140322:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985710720:0@lo:400:kworker.0 00000100:00100000:0.0:1767321554.140381:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321554.140389:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321554.140445:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985709952 00000100:00100000:2.0:1767321554.140457:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710080 00000100:00100000:0.0:1767321554.140458:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985709952 00000100:00100000:2.0:1767321554.140467:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710080 00000100:00100000:2.0:1767321554.140510:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2189 00000100:00100000:2.0:1767321554.140519:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce190a10c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985710080:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321554.140524:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710464 00000100:00100000:0.0:1767321554.140531:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710464 00010000:00020000:2.0:1767321554.140541:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321554.140588:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710336 00000100:00100000:0.0:1767321554.140593:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710336 00000100:00100000:2.0:1767321554.140609:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce190a10c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985710080:12345-0@lo:38:kworker.0 Request processed in 85us (456us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321554.140610:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710592 00000100:00100000:0.0:1767321554.140618:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710592 00000100:00100000:2.0:1767321554.140628:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2189 00000100:00100000:0.0:1767321554.140640:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5891 00000100:00100000:0.0:1767321554.140661:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985710464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321554.140744:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985710464:12345-0@lo:400:kworker.0 Request processed in 91us (493us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321554.140754:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5891 00000100:00100000:0.0:1767321554.140789:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6794 00000100:00100000:0.0:1767321554.140797:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985709952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321554.140815:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321554.140861:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985709952:12345-0@lo:38:kworker.0 Request processed in 64us (879us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321554.140869:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6794 00000100:00100000:0.0:1767321554.140889:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6795 00000100:00100000:0.0:1767321554.140894:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985710336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321554.140904:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321554.140934:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6796 00000100:00100000:0.0:1767321554.140941:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985710336:12345-0@lo:38:kworker.0 Request processed in 42us (662us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321554.140951:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985710592:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321554.140959:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6795 00010000:00020000:2.0:1767321554.140971:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767321554.141023:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321554.141030:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321554.141035:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985710080:0@lo:38:kworker.0 00000100:00100000:1.0:1767321554.141106:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985710848:0@lo:400:kworker.0 00000100:00080000:3.0:1767321554.141136:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321554.141150:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985710208:0@lo:400:kworker.0 00000100:00100000:2.0:1767321554.141867:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985710592:12345-0@lo:38:kworker.0 Request processed in 918us (1483us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321554.141880:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6796 00000100:00100000:2.0:1767321554.142247:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710720 00000100:00100000:2.0:1767321554.142254:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710720 00000100:00100000:2.0:1767321554.142306:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3799 00000100:00100000:2.0:1767321554.142314:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985710720:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321554.142361:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985710720:12345-0@lo:400:kworker.0 Request processed in 49us (1978us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321554.142367:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3799 00000100:00100000:2.0:1767321554.142408:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985710720:0@lo:400:kworker.0 00000100:00100000:2.0:1767321554.142420:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985710464:0@lo:400:kworker.0 00000100:00080000:2.0:1767321554.142464:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321554.143727:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321554.143742:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321554.143747:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5404380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985710336:0@lo:38:kworker.0 00000100:00080000:0.0:1767321554.143766:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321554.143770:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321554.143773:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985710592:0@lo:38:kworker.0 00000100:00080000:0.0:1767321554.143786:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321554.143791:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321554.143794:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985709952:0@lo:38:kworker.0 00000100:00100000:1.0:1767321554.144082:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321554.144813:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710848 00000100:00100000:2.0:1767321554.144827:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710848 00000100:00100000:2.0:1767321554.144979:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3800 00000100:00100000:2.0:1767321554.144993:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985710848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321554.145104:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985710848:12345-0@lo:400:kworker.0 Request processed in 115us (1026us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321554.145112:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3800 00000100:00100000:0.0:1767321554.145857:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985710848:0@lo:400:kworker.0 00000100:00080000:1.0:1767321554.148437:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321554.148704:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321554.148834:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710208 00000100:00100000:2.0:1767321554.148851:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710208 00000100:00100000:2.0:1767321554.148945:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5892 00000100:00100000:2.0:1767321554.148957:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985710208:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321554.149067:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985710208:12345-0@lo:400:kworker.0 Request processed in 112us (370us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321554.149075:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5892 00000100:00100000:0.0:1767321554.149788:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985710208:0@lo:400:kworker.0 00000100:00080000:1.0:1767321554.150037:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767321554.779897:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321555.803596:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321556.827851:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321557.728664:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321557.728693:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985710976:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321557.728739:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985711104:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321557.728776:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321557.728816:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985710976 00000100:00100000:3.0:1767321557.728825:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985710976 00000100:00100000:3.0:1767321557.728889:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3032 00000100:00100000:3.0:1767321557.728898:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985710976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321557.728932:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:1.0:1767321557.729161:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321557.739329:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321557.739351:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985710976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 10451us (10573us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321557.739366:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3032 00000100:00100000:2.0:1767321557.740597:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985710976:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321557.740763:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985711104 00000100:00100000:3.0:1767321557.740772:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985711104 00000100:00100000:3.0:1767321557.741109:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3033 00000100:00100000:3.0:1767321557.741119:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985711104:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321557.741150:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321557.741236:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985711104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (1914us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321557.741244:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3033 00000100:00100000:0.0:1767321557.741412:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985711104:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321557.741436:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767321557.852673:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321558.879236:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321559.213332:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321559.213438:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321559.217728:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920680192 00000100:00100000:0.0:1767321559.217750:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920680192 00000100:00080000:1.0:1767321559.217976:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321559.218048:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920680064 00000100:00100000:0.0:1767321559.218053:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920680064 00000100:00100000:0.0:1767321559.218090:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5893 00000100:00100000:0.0:1767321559.218107:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920680192:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321559.218240:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920680192:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 139us (4921us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321559.218249:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5893 00000100:00100000:0.0:1767321559.218272:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6797 00000100:00100000:0.0:1767321559.218278:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920680064:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321559.218303:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321559.218330:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920680064:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 51us (4892us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321559.218336:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6797 00000100:00080000:0.0:1767321559.261063:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321559.261072:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321559.261076:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321559.261081:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321559.261086:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321559.261097:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6802 00000100:00080000:0.0:1767321559.261102:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321559.261111:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321559.261202:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be8380 x1853163985711232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321559.261228:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321559.261232:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321559.261234:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321559.261238:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6802 00000100:00080000:0.0:1767321559.261240:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321559.261244:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321559.261260:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb100 x1853163985711360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321559.261351:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985711488:0@lo:400:kworker.0 00000100:00100000:1.0:1767321559.261438:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321559.261595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985711232:0@lo:38:kworker.0 00000100:00100000:2.0:1767321559.261630:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321559.261658:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985711360:0@lo:38:kworker.0 00000100:00100000:2.0:1767321559.261821:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321559.261951:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985711488 00000100:00100000:2.0:1767321559.261961:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985711488 00000100:00080000:0.0:1767321559.262060:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321559.262067:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321559.262069:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321559.262075:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6802 00000100:00080000:0.0:1767321559.262077:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321559.262082:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321559.262102:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5be9500 x1853163985711616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321559.262149:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321559.262153:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321559.262155:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321559.262158:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6802 00000100:00080000:0.0:1767321559.262161:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321559.262197:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321559.262210:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5beb800 x1853163985711872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321559.264000:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985711232 00000100:00100000:2.0:1767321559.264013:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985711232 00000100:00100000:2.0:1767321559.264050:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5894 00000100:00100000:2.0:1767321559.264064:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985711488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.265053:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985711360 00000100:00100000:3.0:1767321559.265074:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985711360 00000100:00100000:3.0:1767321559.265139:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2190 00000100:00100000:3.0:1767321559.265149:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985711360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321559.265216:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321559.265306:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985711360:12345-0@lo:38:kworker.0 Request processed in 155us (3491us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321559.265319:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2190 00000100:00100000:3.0:1767321559.265362:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985711616:0@lo:38:kworker.0 00000100:00100000:3.0:1767321559.265413:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321559.265449:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985711872:0@lo:38:kworker.0 00000100:00100000:3.0:1767321559.265471:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321559.265513:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321559.265535:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321559.265540:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985711360:0@lo:38:kworker.0 00000100:00100000:3.0:1767321559.265593:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985711616 00000100:00100000:3.0:1767321559.265597:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985711616 00000100:00100000:3.0:1767321559.265614:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985711872 00000100:00100000:3.0:1767321559.265617:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985711872 00000100:00100000:3.0:1767321559.265627:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6798 00000100:00100000:3.0:1767321559.265632:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985711232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321559.265646:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321559.265681:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985711232:12345-0@lo:38:kworker.0 Request processed in 47us (4051us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321559.265688:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6798 00000100:00100000:3.0:1767321559.265704:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6799 00000100:00100000:3.0:1767321559.265708:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985711616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321559.265719:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321559.265742:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985711616:12345-0@lo:38:kworker.0 Request processed in 32us (329us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321559.265747:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6799 00000100:00100000:3.0:1767321559.265757:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6800 00000100:00100000:3.0:1767321559.265761:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985711872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321559.265770:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321559.265793:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985711872:12345-0@lo:38:kworker.0 Request processed in 31us (321us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321559.265799:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6800 00000100:00080000:3.0:1767321559.265827:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321559.265832:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321559.265835:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985711616:0@lo:38:kworker.0 00000100:00080000:3.0:1767321559.265843:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321559.265847:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321559.265849:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985711872:0@lo:38:kworker.0 00000100:00080000:3.0:1767321559.265859:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321559.265863:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321559.265866:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985711232:0@lo:38:kworker.0 00000100:00080000:3.0:1767321559.265910:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321559.265955:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985711744:0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.265979:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321559.266004:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321559.266010:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985712000:0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.266036:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321559.266065:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985712128:0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.266088:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321559.266117:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985711744 00000100:00100000:3.0:1767321559.266121:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985711744 00000100:00100000:3.0:1767321559.266161:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5895 00000100:00100000:3.0:1767321559.266199:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985711744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.266242:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985711744:12345-0@lo:400:kworker.0 Request processed in 45us (261us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321559.266248:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5895 00000100:00080000:3.0:1767321559.266265:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321559.266285:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5beb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985711744:0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.266318:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712000 00000100:00100000:3.0:1767321559.266321:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712000 00000100:00100000:3.0:1767321559.266364:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712128 00000100:00100000:3.0:1767321559.266368:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712128 00000100:00100000:3.0:1767321559.266389:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3801 00000100:00100000:3.0:1767321559.266395:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985712000:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.266428:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985712000:12345-0@lo:400:kworker.0 Request processed in 34us (392us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321559.266434:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3801 00000100:00100000:3.0:1767321559.266445:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3802 00000100:00100000:3.0:1767321559.266450:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985712128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321559.266476:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985712128:12345-0@lo:400:kworker.0 Request processed in 26us (388us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321559.266482:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3802 00000100:00100000:3.0:1767321559.266506:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985712128:0@lo:400:kworker.0 00000100:00080000:3.0:1767321559.266518:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321559.266521:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321559.266531:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985712000:0@lo:400:kworker.0 00000100:00100000:2.0:1767321559.269340:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985711488:12345-0@lo:400:kworker.0 Request processed in 5267us (7887us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321559.269349:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985711488:0@lo:400:kworker.0 00000100:00100000:2.0:1767321559.269357:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5894 00010000:00010000:2.0:1767321559.903071:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321560.923806:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321561.947461:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321562.843549:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321562.843587:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985712256:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321562.843683:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321562.843928:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712256 00000100:00100000:1.0:1767321562.843940:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712256 00000100:00100000:0.0:1767321562.843996:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985712384:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321562.844025:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3034 00000100:00100000:0.0:1767321562.844041:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321562.844042:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985712256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321562.844083:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321562.844195:0:18374:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712384 00000100:00100000:2.0:1767321562.844199:0:18374:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712384 00000100:00100000:2.0:1767321562.844227:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3035 00000100:00100000:2.0:1767321562.844235:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a7bb3c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985712384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321562.844252:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00080000:3.0:1767321562.844359:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321562.844364:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321562.857106:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a7bb3c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985712384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 12859us (13054us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321562.857123:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3035 00000100:00100000:1.0:1767321562.857370:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985712256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 13336us (13687us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321562.857378:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3034 00000100:00100000:2.0:1767321562.872226:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985712256:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321562.876441:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985712384:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321562.973064:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321563.995549:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321564.330035:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321564.330694:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920680448 00000100:00100000:2.0:1767321564.330715:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920680448 00000100:00100000:0.0:1767321564.330820:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321564.330828:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5896 00000100:00100000:2.0:1767321564.330841:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920680448:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321564.330897:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920680320 00000100:00100000:1.0:1767321564.330903:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920680320 00000100:00100000:1.0:1767321564.330930:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6801 00000100:00100000:1.0:1767321564.330937:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920680320:12345-192.168.204.26@tcp:38:kworker.0 00000100:00100000:2.0:1767321564.330952:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920680448:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 115us (928us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321564.330960:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5896 00010000:00020000:1.0:1767321564.330967:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321564.331000:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920680320:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 63us (186us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321564.331007:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6801 00000100:00080000:3.0:1767321564.331107:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321564.379983:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321564.379998:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321564.380001:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321564.380007:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321564.380013:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321564.380023:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6807 00000100:00080000:2.0:1767321564.380030:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767321564.380038:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321564.380095:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405180 x1853163985712512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321564.380133:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321564.380137:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321564.380139:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321564.380142:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6807 00000100:00080000:2.0:1767321564.380146:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767321564.380150:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321564.380185:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5407800 x1853163985712640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321564.380238:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321564.380242:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321564.380245:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321564.380249:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6807 00000100:00080000:2.0:1767321564.380252:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767321564.380255:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321564.380320:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985712768:0@lo:400:kworker.0 00000100:00100000:3.0:1767321564.380416:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321564.380443:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405500 x1853163985712896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321564.380509:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321564.380513:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321564.380514:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321564.380518:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6807 00000100:00080000:2.0:1767321564.380521:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767321564.380526:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321564.380539:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5406a00 x1853163985713152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321564.380599:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985713024:0@lo:400:kworker.0 00000100:00100000:3.0:1767321564.380617:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321564.380630:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985713408:0@lo:400:kworker.0 00000100:00100000:3.0:1767321564.380670:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321564.380713:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985713408 00000100:00100000:3.0:1767321564.380722:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985713408 00000100:00100000:3.0:1767321564.380792:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3803 00000100:00100000:2.0:1767321564.380802:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321564.380804:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985713408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321564.380864:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985712512:0@lo:38:kworker.0 00000100:00100000:3.0:1767321564.380876:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985713408:12345-0@lo:400:kworker.0 Request processed in 73us (203us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321564.380886:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3803 00000100:00080000:3.0:1767321564.380920:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321564.380947:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985713280:0@lo:400:kworker.0 00000100:00100000:3.0:1767321564.380971:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985713408:0@lo:400:kworker.0 00000100:00100000:0.0:1767321564.380991:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321564.381037:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985713280 00000100:00100000:3.0:1767321564.381043:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985713280 00000100:00100000:3.0:1767321564.381087:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3804 00000100:00100000:3.0:1767321564.381096:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985713280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321564.381144:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985713280:12345-0@lo:400:kworker.0 Request processed in 50us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321564.381151:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3804 00000100:00080000:3.0:1767321564.381203:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321564.381218:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985713280:0@lo:400:kworker.0 00000100:00100000:1.0:1767321564.385111:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712768 00000100:00100000:1.0:1767321564.385127:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712768 00000100:00080000:3.0:1767321564.385295:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321564.387815:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985713024 00000100:00100000:0.0:1767321564.387838:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985713024 00000100:00100000:1.0:1767321564.387996:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5897 00000100:00100000:0.0:1767321564.388010:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5898 00000100:00100000:0.0:1767321564.388026:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985713024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321564.388028:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985712768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321564.388191:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985713024:12345-0@lo:400:kworker.0 Request processed in 142us (7363us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321564.388201:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5898 00000100:00100000:1.0:1767321564.388306:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4dfb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985712768:12345-0@lo:400:kworker.0 Request processed in 295us (7903us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321564.388325:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5897 00000100:00100000:2.0:1767321564.388330:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321564.393709:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712512 00000100:00100000:3.0:1767321564.393723:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712512 00000100:00100000:2.0:1767321564.393745:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985712640:0@lo:38:kworker.0 00000100:00100000:3.0:1767321564.393802:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6802 00000100:00100000:3.0:1767321564.393813:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985712512:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321564.393837:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321564.393869:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985712896:0@lo:38:kworker.0 00000100:00100000:2.0:1767321564.393893:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321564.393903:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985713152:0@lo:38:kworker.0 00000100:00100000:2.0:1767321564.393922:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767321564.393975:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321564.393977:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712896 00000100:00100000:2.0:1767321564.393986:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712896 00000100:00100000:2.0:1767321564.394013:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985713152 00000100:00100000:2.0:1767321564.394016:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985713152 00000100:00100000:2.0:1767321564.394035:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6803 00000100:00100000:2.0:1767321564.394045:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985712896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321564.394068:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321564.394075:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5404000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985712512:12345-0@lo:38:kworker.0 Request processed in 261us (5753us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321564.394084:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6802 00000100:00100000:3.0:1767321564.394104:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6804 00000100:00100000:3.0:1767321564.394109:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985713152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321564.394121:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321564.394147:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985713152:12345-0@lo:38:kworker.0 Request processed in 37us (225us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321564.394152:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6804 00000100:00100000:3.0:1767321564.394242:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985712768:0@lo:400:kworker.0 00000100:00100000:2.0:1767321564.394318:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985712896:12345-0@lo:38:kworker.0 Request processed in 271us (421us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321564.394328:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6803 00000100:00100000:2.0:1767321564.394367:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985712640 00000100:00100000:2.0:1767321564.394372:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985712640 00000100:00100000:2.0:1767321564.394414:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2191 00000100:00100000:2.0:1767321564.394420:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985712640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321564.394433:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321564.394453:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985713024:0@lo:400:kworker.0 00000100:00100000:2.0:1767321564.394491:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985712640:12345-0@lo:38:kworker.0 Request processed in 67us (653us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321564.394499:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2191 00000100:00080000:3.0:1767321564.394532:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321564.394541:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321564.394557:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985712512:0@lo:38:kworker.0 00000100:00080000:2.0:1767321564.394561:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321564.394576:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321564.394581:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321564.394585:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985712640:0@lo:38:kworker.0 00000100:00080000:3.0:1767321564.394594:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321564.394598:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321564.394601:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985712896:0@lo:38:kworker.0 00000100:00080000:3.0:1767321564.394609:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321564.394612:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321564.394614:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985713152:0@lo:38:kworker.0 00010000:00010000:1.0:1767321565.019481:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321566.044483:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321567.068051:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321567.963542:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321567.963564:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985713536:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321567.963632:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321567.963809:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985713664:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321567.963840:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321567.969790:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985713536 00000100:00100000:1.0:1767321567.969804:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985713536 00000100:00100000:1.0:1767321567.969908:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985713664 00000100:00100000:1.0:1767321567.969913:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985713664 00000100:00100000:2.0:1767321567.970559:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3036 00000100:00100000:2.0:1767321567.970574:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985713536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321567.970607:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321567.970772:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985713536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 203us (7139us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321567.970780:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3036 00000100:00100000:2.0:1767321567.970847:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985713536:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321567.972875:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321567.972885:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321567.973240:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3037 00000100:00100000:1.0:1767321567.973451:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985713664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321567.973488:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321567.973566:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985713664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 125us (9725us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321567.973650:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985713664:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321567.979562:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3037 00010000:00010000:1.0:1767321568.091442:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767321568.426321:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920680576 offset 0 length 520: 4 00000100:00100000:0.0:1767321568.432824:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321568.432927:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321568.433484:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920680704 00000100:00100000:3.0:1767321568.433495:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920680704 00000100:00100000:3.0:1767321568.433543:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920680832 00000100:00100000:3.0:1767321568.433546:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920680832 00000100:00100000:3.0:1767321568.433587:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5899 00000100:00100000:3.0:1767321568.433597:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920680832:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321568.433661:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920680832:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 66us (733us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321568.433668:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5899 00000100:00100000:3.0:1767321568.433696:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6805 00000100:00100000:3.0:1767321568.433701:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920680704:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321568.433719:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321568.433741:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920680704:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 39us (927us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321568.433747:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6805 00000100:00080000:0.0:1767321568.434221:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321568.475293:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321543/real 1767321543] req@ffff8ce1b5bead80 x1853163985708288/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321568 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321568.475426:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1b5bead80 x1853163985708288/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321568 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321568.475448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321568.475584:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321568.475589:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985708288:0@lo:250:kworker.0 00000100:00080000:3.0:1767321568.481573:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321568.481583:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321568.481587:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321568.481595:0:44026:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6792 00000100:00080000:3.0:1767321568.481600:0:44026:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321568.481607:0:44026:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321568.481643:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985713792/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321568.481669:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321568.481684:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321568.481686:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321568.481690:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6812 00000100:00080000:3.0:1767321568.481693:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767321568.481696:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321568.481711:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5406a00 x1853163985713920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321568.481720:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321568.481723:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321568.481725:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321568.481728:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6812 00000100:00080000:3.0:1767321568.481730:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767321568.481734:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321568.481746:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405500 x1853163985714048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321568.481777:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321568.481781:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321568.481783:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321568.481786:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6812 00000100:00080000:3.0:1767321568.481789:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767321568.481791:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321568.481802:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405180 x1853163985714304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321568.481826:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321568.481829:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321568.481830:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321568.481833:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6812 00000100:00080000:3.0:1767321568.481835:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767321568.481838:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321568.481847:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a5405f80 x1853163985714560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321568.481927:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321568.481945:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985714432:0@lo:400:kworker.0 00000100:00100000:1.0:1767321568.482015:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321568.482076:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985713792:0@lo:250:kworker.0 00000400:00000100:1.0:1767321568.482095:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985713792 offset 0 length 520: 4 00000100:00100000:1.0:1767321568.482113:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5406a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985713920:0@lo:38:kworker.0 00000100:00100000:1.0:1767321568.482131:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321568.482150:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985714048:0@lo:38:kworker.0 00000100:00100000:1.0:1767321568.482190:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321568.482209:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985714304:0@lo:38:kworker.0 00000100:00100000:1.0:1767321568.482227:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321568.482246:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985714560:0@lo:38:kworker.0 00000100:00100000:1.0:1767321568.482264:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321568.482296:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714048 00000100:00100000:1.0:1767321568.482303:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714048 00000100:00100000:1.0:1767321568.482346:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2192 00000100:00100000:1.0:1767321568.482352:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985714048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321568.482372:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321568.482419:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985714048:12345-0@lo:38:kworker.0 Request processed in 66us (228us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321568.482426:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2192 00000100:00100000:1.0:1767321568.482453:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714432 00000100:00100000:1.0:1767321568.482456:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714432 00000100:00100000:1.0:1767321568.482490:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985713920 00000100:00100000:1.0:1767321568.482493:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985713920 00000100:00100000:1.0:1767321568.482505:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714304 00000100:00100000:1.0:1767321568.482507:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714304 00000100:00100000:1.0:1767321568.482514:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714560 00000100:00100000:1.0:1767321568.482516:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714560 00000100:00100000:1.0:1767321568.482523:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5900 00000100:00100000:1.0:1767321568.482529:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985714432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321568.482568:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985714432:12345-0@lo:400:kworker.0 Request processed in 39us (553us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321568.482574:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5900 00000100:00100000:1.0:1767321568.482584:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6806 00000100:00100000:1.0:1767321568.482588:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985713920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321568.482596:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321568.482618:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985713920:12345-0@lo:38:kworker.0 Request processed in 29us (485us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321568.482623:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6806 00000100:00100000:1.0:1767321568.482633:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6807 00000100:00100000:1.0:1767321568.482636:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985714304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321568.482644:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321568.482664:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985714304:12345-0@lo:38:kworker.0 Request processed in 26us (436us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321568.482670:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6807 00000100:00100000:1.0:1767321568.482679:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6808 00000100:00100000:1.0:1767321568.482683:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985714560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321568.482691:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321568.482712:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985714560:12345-0@lo:38:kworker.0 Request processed in 27us (447us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321568.482717:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6808 00000100:00100000:1.0:1767321568.482756:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985714432:0@lo:400:kworker.0 00000100:00080000:1.0:1767321568.482785:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321568.482790:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321568.482793:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5406a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985713920:0@lo:38:kworker.0 00000100:00080000:1.0:1767321568.482800:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321568.482802:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321568.482804:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985714048:0@lo:38:kworker.0 00000100:00080000:1.0:1767321568.482811:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321568.482814:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321568.482817:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985714304:0@lo:38:kworker.0 00000100:00080000:1.0:1767321568.482824:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321568.482827:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321568.482830:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5405f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985714560:0@lo:38:kworker.0 00000100:00080000:1.0:1767321568.482852:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321568.482884:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321568.482891:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985714176:0@lo:400:kworker.0 00000100:00100000:1.0:1767321568.482915:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321568.482943:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714176 00000100:00100000:1.0:1767321568.482948:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714176 00000100:00080000:1.0:1767321568.482962:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321568.482984:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5901 00000100:00100000:1.0:1767321568.482991:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985714176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321568.483021:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985714176:12345-0@lo:400:kworker.0 Request processed in 31us (105us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321568.483026:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5901 00000100:00100000:1.0:1767321568.483049:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985714176:0@lo:400:kworker.0 00010000:00010000:2.0:1767321569.115447:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321570.140248:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321571.163652:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321572.187400:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321573.083857:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321573.083871:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985714688:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321573.083924:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321573.083957:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714688 00000100:00100000:0.0:1767321573.083966:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714688 00000100:00100000:3.0:1767321573.083981:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321573.083988:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985714816:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321573.084006:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321573.084018:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714816 00000100:00100000:0.0:1767321573.084023:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714816 00000100:00100000:3.0:1767321573.084034:0:18374:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3038 00000100:00100000:3.0:1767321573.084044:0:18374:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985714688:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321573.084053:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3039 00000100:00100000:0.0:1767321573.084063:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985714816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321573.084086:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00002000:00000024:3.0:1767321573.084087:0:18374:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321573.084136:0:18374:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985714688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321573.084144:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985714816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (135us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321573.084149:0:18374:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3038 00000100:00100000:0.0:1767321573.084158:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3039 00000100:00100000:3.0:1767321573.084227:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a5407800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985714688:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321573.084229:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985714816:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321573.084270:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321573.084274:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767321573.211450:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321573.547954:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321573.548004:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321573.548121:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920681088 00000100:00100000:1.0:1767321573.548131:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920681088 00000100:00100000:1.0:1767321573.548242:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920680960 00000100:00100000:1.0:1767321573.548246:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920680960 00000100:00100000:1.0:1767321573.548268:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5902 00000100:00100000:1.0:1767321573.548289:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920681088:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321573.548367:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6809 00000100:00100000:2.0:1767321573.548375:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920680960:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321573.548406:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767321573.548461:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321573.548520:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920681088:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 242us (574us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321573.548528:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5902 00000100:00100000:2.0:1767321573.549718:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5405500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920680960:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 1335us (1705us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321573.549844:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6809 00000100:00080000:1.0:1767321573.595625:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321573.595633:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321573.595637:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321573.595641:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321573.595645:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321573.595653:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6816 00000100:00080000:1.0:1767321573.595658:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767321573.595665:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321573.595705:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73f800 x1853163985714944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321573.595730:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321573.595734:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321573.595736:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321573.595740:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6816 00000100:00080000:1.0:1767321573.595742:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767321573.595745:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321573.595759:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ca80 x1853163985715072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321573.595794:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321573.595798:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321573.595800:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321573.595803:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6816 00000100:00080000:1.0:1767321573.595807:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00100000:2.0:1767321573.595807:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767321573.595812:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321573.595825:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985715200:0@lo:400:kworker.0 00000100:00100000:1.0:1767321573.595828:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bdaa00 x1853163985715328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321573.595857:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321573.595861:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321573.595863:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321573.595866:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6816 00000100:00080000:1.0:1767321573.595869:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767321573.595873:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321573.595884:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b5bd9880 x1853163985715584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321573.595912:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321573.595934:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985714944:0@lo:38:kworker.0 00000100:00100000:3.0:1767321573.596087:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321573.596098:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985715456:0@lo:400:kworker.0 00000100:00100000:3.0:1767321573.596129:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321573.596154:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715200 00000100:00100000:2.0:1767321573.596185:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715200 00000100:00100000:3.0:1767321573.596204:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321573.596215:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321573.596234:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985715072:0@lo:38:kworker.0 00000100:00100000:2.0:1767321573.596264:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715456 00000100:00100000:3.0:1767321573.596264:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321573.596269:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715456 00000100:00100000:3.0:1767321573.596302:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985715328:0@lo:38:kworker.0 00000100:00100000:2.0:1767321573.596314:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985714944 00000100:00100000:2.0:1767321573.596319:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985714944 00000100:00100000:3.0:1767321573.596328:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321573.596338:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715328 00000100:00100000:2.0:1767321573.596342:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715328 00000100:00100000:3.0:1767321573.596347:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985715584:0@lo:38:kworker.0 00000100:00100000:2.0:1767321573.596357:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5903 00000100:00100000:2.0:1767321573.596367:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985715200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321573.596374:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321573.596416:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715584 00000100:00100000:3.0:1767321573.596420:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715584 00000100:00100000:2.0:1767321573.596426:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af18c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985715200:12345-0@lo:400:kworker.0 Request processed in 59us (512us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321573.596435:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5903 00000100:00100000:3.0:1767321573.596441:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5904 00000100:00100000:3.0:1767321573.596449:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985715456:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321573.596458:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6810 00000100:00100000:2.0:1767321573.596464:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985714944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321573.596482:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321573.596488:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5406a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985715456:12345-0@lo:400:kworker.0 Request processed in 39us (358us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321573.596495:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5904 00000100:00100000:3.0:1767321573.596536:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985715456:0@lo:400:kworker.0 00000100:00100000:3.0:1767321573.596562:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6811 00000100:00100000:3.0:1767321573.596566:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985715328:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321573.596577:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321573.597384:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6812 00000100:00100000:0.0:1767321573.597674:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985715584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321573.597687:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321573.597772:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715072 00000100:00100000:1.0:1767321573.597779:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715072 00000100:00100000:1.0:1767321573.597810:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2193 00000100:00100000:1.0:1767321573.597815:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985715072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321573.597827:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321573.597829:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5407480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985714944:12345-0@lo:38:kworker.0 Request processed in 1361us (1628us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321573.597839:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6810 00000100:00080000:2.0:1767321573.597864:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321573.597877:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985715072:12345-0@lo:38:kworker.0 Request processed in 59us (1613us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321573.597884:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2193 00000100:00080000:2.0:1767321573.597908:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321573.597914:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321573.597927:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985714944:0@lo:38:kworker.0 00000100:00080000:2.0:1767321573.598221:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321573.598226:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321573.598230:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985715072:0@lo:38:kworker.0 00000100:00100000:2.0:1767321573.598276:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985715200:0@lo:400:kworker.0 00000100:00100000:3.0:1767321573.598276:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985715328:12345-0@lo:38:kworker.0 Request processed in 1705us (1945us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321573.598289:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6811 00000100:00080000:2.0:1767321573.598318:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321573.598323:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321573.598327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985715328:0@lo:38:kworker.0 00000100:00100000:0.0:1767321573.605975:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985715584:12345-0@lo:38:kworker.0 Request processed in 8292us (9594us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321573.605995:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6812 00000100:00080000:3.0:1767321573.606349:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321573.606356:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321573.606360:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985715584:0@lo:38:kworker.0 00010000:00010000:1.0:1767321574.235438:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321575.259623:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321576.283753:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321577.308559:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321578.203540:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321578.203557:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985715712:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321578.203631:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321578.203938:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715712 00000100:00100000:2.0:1767321578.203947:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715712 00000100:00100000:2.0:1767321578.204020:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3040 00000100:00100000:2.0:1767321578.204039:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985715712:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321578.204069:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321578.204114:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985715712:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (483us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321578.204122:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3040 00000100:00080000:3.0:1767321578.204271:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321578.205218:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985715712:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321578.205849:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321578.205865:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985715840:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321578.205915:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321578.206068:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715840 00000100:00100000:0.0:1767321578.206075:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715840 00000100:00100000:0.0:1767321578.206130:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3041 00000100:00100000:0.0:1767321578.206137:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985715840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321578.206157:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321578.206202:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985715840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321578.206219:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3041 00000100:00100000:0.0:1767321578.206255:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985715840:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321578.206466:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321578.331364:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321578.669652:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321578.669764:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321578.669987:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920681216 00000100:00100000:0.0:1767321578.669997:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920681216 00000100:00100000:2.0:1767321578.670051:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920681344 00000100:00100000:2.0:1767321578.670056:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920681344 00000100:00100000:2.0:1767321578.670115:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5905 00000100:00100000:2.0:1767321578.670127:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920681344:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321578.670261:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920681344:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 137us (497us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321578.670270:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5905 00000100:00100000:2.0:1767321578.670290:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6813 00000100:00100000:2.0:1767321578.670294:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920681216:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321578.670315:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321578.670339:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2512:x1853163920681216:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 44us (700us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321578.670345:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6813 00000100:00080000:3.0:1767321578.670635:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321578.715576:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321578.715586:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321578.715590:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321578.715594:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321578.715598:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321578.715607:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6821 00000100:00080000:2.0:1767321578.715611:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321578.715618:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321578.715662:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e8e3800 x1853163985715968/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321578.715792:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985715968:0@lo:38:kworker.0 00000100:00100000:3.0:1767321578.715863:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321578.716037:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985715968 00000100:00100000:1.0:1767321578.716048:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985715968 00000100:00100000:1.0:1767321578.716094:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6814 00000100:00100000:1.0:1767321578.716102:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985715968:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321578.716125:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321578.716297:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321578.716302:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321578.716304:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321578.716308:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6821 00000100:00080000:2.0:1767321578.716311:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321578.716315:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321578.716332:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1af73ca80 x1853163985716096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321578.716412:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985716096:0@lo:38:kworker.0 00000100:00100000:3.0:1767321578.716442:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321578.716603:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716096 00000100:00100000:0.0:1767321578.716608:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716096 00000100:00100000:0.0:1767321578.716634:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2194 00000100:00100000:0.0:1767321578.716639:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985716096:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321578.716650:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321578.716694:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985716096:12345-0@lo:38:kworker.0 Request processed in 53us (251us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321578.716703:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2194 00000100:00100000:1.0:1767321578.716874:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985715968:12345-0@lo:38:kworker.0 Request processed in 770us (1009us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321578.716881:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6814 00000100:00100000:0.0:1767321578.716984:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985716224:0@lo:400:kworker.0 00000100:00100000:0.0:1767321578.717011:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321578.717069:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716224 00000100:00100000:1.0:1767321578.717074:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716224 00000100:00100000:1.0:1767321578.717116:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5906 00000100:00100000:1.0:1767321578.717122:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985716224:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321578.717249:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321578.717253:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321578.717256:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321578.717260:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6821 00000100:00080000:2.0:1767321578.717263:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321578.717267:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321578.717284:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633480 x1853163985716352/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321578.717309:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321578.717312:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321578.717314:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321578.717316:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6821 00000100:00080000:2.0:1767321578.717318:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321578.717322:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321578.717332:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633100 x1853163985716608/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321578.717365:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321578.717381:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321578.717390:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985716480:0@lo:400:kworker.0 00000100:00100000:2.0:1767321578.717415:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321578.717442:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716480 00000100:00100000:2.0:1767321578.717446:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716480 00000100:00100000:2.0:1767321578.717489:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5907 00000100:00100000:2.0:1767321578.717496:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985716480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321578.717532:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985716480:12345-0@lo:400:kworker.0 Request processed in 37us (116us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321578.717538:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5907 00000100:00100000:2.0:1767321578.717575:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188630000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985716480:0@lo:400:kworker.0 00000100:00080000:2.0:1767321578.717592:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321578.717844:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321578.717850:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321578.717853:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985715968:0@lo:38:kworker.0 00000100:00100000:3.0:1767321578.717870:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985716352:0@lo:38:kworker.0 00000100:00100000:3.0:1767321578.717894:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321578.717921:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985716608:0@lo:38:kworker.0 00000100:00100000:3.0:1767321578.717940:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321578.717975:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321578.717980:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321578.717982:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985716096:0@lo:38:kworker.0 00000100:00100000:2.0:1767321578.718085:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716352 00000100:00100000:2.0:1767321578.718090:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716352 00000100:00100000:2.0:1767321578.718115:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716608 00000100:00100000:2.0:1767321578.718118:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716608 00000100:00100000:2.0:1767321578.718135:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6815 00000100:00100000:2.0:1767321578.718140:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985716352:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321578.718155:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321578.718219:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985716352:12345-0@lo:38:kworker.0 Request processed in 77us (323us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321578.718226:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6815 00000100:00100000:2.0:1767321578.718238:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6816 00000100:00100000:2.0:1767321578.718242:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985716608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321578.718251:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321578.718274:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5beb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985716608:12345-0@lo:38:kworker.0 Request processed in 30us (332us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321578.718279:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6816 00000100:00080000:3.0:1767321578.718365:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321578.718369:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321578.718372:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985716352:0@lo:38:kworker.0 00000100:00080000:3.0:1767321578.718379:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321578.718382:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321578.718384:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985716608:0@lo:38:kworker.0 00000100:00100000:1.0:1767321578.718696:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985716224:12345-0@lo:400:kworker.0 Request processed in 1573us (1682us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321578.718704:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5906 00000100:00100000:0.0:1767321578.718821:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1af73f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985716224:0@lo:400:kworker.0 00010000:00010000:0.0:1767321579.355442:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321580.379680:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321581.405294:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321582.429210:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321583.323833:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321583.323861:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985716736:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321583.323971:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321583.324391:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716736 00000100:00100000:2.0:1767321583.324403:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716736 00000100:00100000:2.0:1767321583.324498:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3042 00000100:00100000:2.0:1767321583.324514:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985716736:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321583.324515:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:2.0:1767321583.324567:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321583.324643:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985716736:12345-0@lo:13:osp-pre-1-1.0 Request processed in 135us (672us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321583.324652:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3042 00000100:00100000:0.0:1767321583.324889:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985716736:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321583.325449:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985716864:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321583.325502:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321583.325609:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716864 00000100:00100000:1.0:1767321583.325617:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716864 00000100:00100000:1.0:1767321583.325683:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3043 00000100:00100000:1.0:1767321583.325692:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985716864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321583.325717:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321583.325762:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188630000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985716864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321583.325769:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3043 00000100:00100000:1.0:1767321583.325815:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985716864:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767321583.325884:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767321583.454228:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321583.786910:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321583.787987:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920681472 00000100:00100000:0.0:1767321583.788004:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920681472 00000100:00100000:3.0:1767321583.788257:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6817 00000100:00100000:3.0:1767321583.788270:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920681472:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:3.0:1767321583.788319:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321583.788584:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920681472:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 312us (1693us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321583.788594:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6817 00000100:00100000:1.0:1767321583.797407:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321583.797672:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920681600 00000100:00100000:0.0:1767321583.797688:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920681600 00000100:00100000:0.0:1767321583.797801:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5908 00000100:00100000:0.0:1767321583.797814:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920681600:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321583.797938:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920681600:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 125us (545us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321583.797947:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5908 00000100:00080000:2.0:1767321583.798633:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321583.836023:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321583.836039:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321583.836042:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321583.836047:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321583.836052:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321583.836064:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6826 00000100:00080000:2.0:1767321583.836070:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321583.836078:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321583.836187:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985716992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321583.836217:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321583.836221:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321583.836223:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321583.836228:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6826 00000100:00080000:2.0:1767321583.836231:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321583.836235:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321583.836247:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985716992:0@lo:38:kworker.0 00000100:00100000:2.0:1767321583.836254:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633b80 x1853163985717120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321583.836348:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321583.836439:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985716992 00000100:00100000:0.0:1767321583.836440:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985717120:0@lo:38:kworker.0 00000100:00100000:3.0:1767321583.836453:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985716992 00000100:00100000:0.0:1767321583.836475:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321583.836516:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6818 00000100:00100000:3.0:1767321583.836527:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985716992:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321583.836541:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985717120 00000100:00100000:0.0:1767321583.836548:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985717120 00010000:00020000:3.0:1767321583.836556:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321583.836617:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985716992:12345-0@lo:38:kworker.0 Request processed in 91us (269us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321583.836626:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6818 00000100:00100000:0.0:1767321583.836669:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2195 00000100:00100000:0.0:1767321583.836675:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985717120:12345-0@lo:38:kworker.0 00000100:00080000:3.0:1767321583.836680:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321583.836688:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:0.0:1767321583.836689:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321583.836696:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985716992:0@lo:38:kworker.0 00000100:00100000:0.0:1767321583.836739:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985717120:12345-0@lo:38:kworker.0 Request processed in 62us (263us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767321583.836744:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767321583.836748:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2195 00000100:00080000:3.0:1767321583.836752:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321583.836757:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985717120:0@lo:38:kworker.0 00000100:00080000:2.0:1767321583.837550:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321583.837557:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321583.837560:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321583.837567:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6826 00000100:00080000:2.0:1767321583.837571:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321583.837576:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321583.837600:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633480 x1853163985717376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321583.837650:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321583.837654:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321583.837655:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321583.837659:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6826 00000100:00080000:2.0:1767321583.837662:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321583.837665:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321583.837675:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985717632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321583.837723:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985717760:0@lo:400:kworker.0 00000100:00100000:2.0:1767321583.837780:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321583.837819:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321583.837825:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985717504:0@lo:400:kworker.0 00000100:00100000:2.0:1767321583.837850:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321583.837885:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985717760 00000100:00100000:0.0:1767321583.837974:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985717248:0@lo:400:kworker.0 00000100:00100000:0.0:1767321583.838017:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321583.838038:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321583.838043:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985717888:0@lo:400:kworker.0 00000100:00100000:0.0:1767321583.838182:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321583.838252:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985717504 00000100:00100000:0.0:1767321583.838260:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985717504 00000100:00100000:0.0:1767321583.838318:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985717248 00000100:00100000:0.0:1767321583.838321:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985717248 00000100:00100000:0.0:1767321583.838352:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5909 00000100:00100000:0.0:1767321583.838362:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985717504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321583.838368:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985717888 00000100:00100000:3.0:1767321583.838375:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985717888 00000100:00100000:0.0:1767321583.838423:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985717504:12345-0@lo:400:kworker.0 Request processed in 62us (570us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321583.838431:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5909 00000100:00100000:0.0:1767321583.838449:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5910 00000100:00100000:0.0:1767321583.838455:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985717248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321583.838488:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985717248:12345-0@lo:400:kworker.0 Request processed in 33us (469us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321583.838494:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5910 00000100:00100000:0.0:1767321583.838542:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985717248:0@lo:400:kworker.0 00000100:00100000:3.0:1767321583.838571:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3805 00000100:00100000:3.0:1767321583.838581:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985717888:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321583.838608:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321583.838635:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985717888:12345-0@lo:400:kworker.0 Request processed in 56us (451us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321583.838636:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985717504:0@lo:400:kworker.0 00000100:00100000:3.0:1767321583.838649:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3805 00000100:00100000:1.0:1767321583.838748:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985717376:0@lo:38:kworker.0 00000100:00100000:1.0:1767321583.838795:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321583.838812:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985717632:0@lo:38:kworker.0 00000100:00100000:1.0:1767321583.838836:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321583.838857:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985717376 00000100:00100000:1.0:1767321583.838860:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985717376 00000100:00100000:1.0:1767321583.838878:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985717632 00000100:00100000:1.0:1767321583.838880:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985717632 00000100:00100000:1.0:1767321583.838894:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6819 00000100:00100000:1.0:1767321583.838899:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985717376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321583.838918:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321583.838954:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985717376:12345-0@lo:38:kworker.0 Request processed in 53us (157us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321583.838960:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6819 00000100:00100000:1.0:1767321583.838972:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6820 00000100:00100000:1.0:1767321583.838975:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985717632:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321583.838982:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321583.839002:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985717632:12345-0@lo:38:kworker.0 Request processed in 25us (165us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321583.839007:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6820 00000100:00080000:1.0:1767321583.839031:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321583.839036:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321583.839040:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985717376:0@lo:38:kworker.0 00000100:00080000:1.0:1767321583.840665:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321583.840682:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321583.840688:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985717632:0@lo:38:kworker.0 00000100:00100000:2.0:1767321583.840726:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985717760 00000100:00100000:3.0:1767321583.841546:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985717888:0@lo:400:kworker.0 00000100:00100000:2.0:1767321583.848202:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3806 00000100:00100000:2.0:1767321583.848245:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985717760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321583.848428:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985717760:12345-0@lo:400:kworker.0 Request processed in 200us (10637us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321583.848438:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3806 00000100:00100000:1.0:1767321583.848591:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985717760:0@lo:400:kworker.0 00010000:00010000:2.0:1767321584.475515:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321585.499484:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321586.526324:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321587.547981:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321588.443708:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321588.443735:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985718016:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321588.443836:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321588.444014:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718016 00000100:00100000:1.0:1767321588.444054:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718016 00000100:00100000:1.0:1767321588.444143:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3044 00000100:00100000:1.0:1767321588.444155:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985718016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321588.444223:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321588.444287:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985718016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 133us (449us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321588.444296:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3044 00000100:00080000:0.0:1767321588.444361:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321588.444450:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985718016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321588.444791:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985718144:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321588.444840:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321588.444937:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718144 00000100:00100000:2.0:1767321588.444944:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718144 00000100:00100000:2.0:1767321588.444999:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3045 00000100:00100000:2.0:1767321588.445008:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985718144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321588.445038:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321588.445081:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985718144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (240us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321588.445087:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3045 00000100:00080000:0.0:1767321588.445150:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321588.445199:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b5be8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985718144:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321588.571464:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321588.907509:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321588.907614:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321588.915265:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920681856 00000100:00100000:1.0:1767321588.915279:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920681856 00000100:00100000:1.0:1767321588.915493:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920681728 00000100:00100000:1.0:1767321588.915498:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920681728 00000100:00100000:1.0:1767321588.915520:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5911 00000100:00100000:1.0:1767321588.915533:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920681856:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321588.915616:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5be8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920681856:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 87us (8118us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321588.915625:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5911 00000100:00100000:1.0:1767321588.915644:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6821 00000100:00100000:1.0:1767321588.915648:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920681728:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321588.915668:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321588.915696:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920681728:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 46us (8082us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321588.915702:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6821 00000100:00080000:3.0:1767321588.920312:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321588.958457:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321588.958468:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321588.958471:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321588.958476:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321588.958480:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321588.958488:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6832 00000100:00080000:0.0:1767321588.958493:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321588.958500:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321588.958539:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524f100 x1853163985718272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321588.958558:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321588.958561:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321588.958563:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321588.958566:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6832 00000100:00080000:0.0:1767321588.958569:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321588.958583:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321588.958597:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524fb80 x1853163985718400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321588.958646:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321588.958650:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321588.958652:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321588.958656:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6832 00000100:00080000:0.0:1767321588.958659:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321588.958662:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321588.958682:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985718656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321588.958710:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321588.958713:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321588.958715:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321588.958719:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6832 00000100:00080000:0.0:1767321588.958721:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321588.958724:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321588.958734:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633480 x1853163985718912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321588.958770:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985719040:0@lo:400:kworker.0 00000100:00100000:0.0:1767321588.958833:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321588.958895:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321588.958901:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985718784:0@lo:400:kworker.0 00000100:00100000:0.0:1767321588.958924:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321588.958968:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985719040 00000100:00100000:0.0:1767321588.958975:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985719040 00000100:00100000:0.0:1767321588.959030:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3807 00000100:00100000:0.0:1767321588.959039:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985719040:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321588.959087:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985719040:12345-0@lo:400:kworker.0 Request processed in 49us (253us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321588.959096:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3807 00000100:00100000:0.0:1767321588.959140:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985719040:0@lo:400:kworker.0 00000100:00100000:3.0:1767321588.959492:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985719168:0@lo:400:kworker.0 00000100:00100000:3.0:1767321588.959525:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321588.959555:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321588.959561:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985718528:0@lo:400:kworker.0 00000100:00100000:3.0:1767321588.959581:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321588.959677:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985718272:0@lo:38:kworker.0 00000100:00100000:1.0:1767321588.959707:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321588.959739:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985718400:0@lo:38:kworker.0 00000100:00100000:1.0:1767321588.959761:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321588.959784:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985718656:0@lo:38:kworker.0 00000100:00100000:1.0:1767321588.959802:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321588.959816:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985718912:0@lo:38:kworker.0 00000100:00100000:1.0:1767321588.959834:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321588.959861:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718784 00000100:00100000:1.0:1767321588.959866:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718784 00000100:00100000:1.0:1767321588.959902:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718528 00000100:00100000:1.0:1767321588.959905:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718528 00000100:00100000:1.0:1767321588.959915:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718272 00000100:00100000:1.0:1767321588.959918:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718272 00000100:00100000:1.0:1767321588.959925:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718656 00000100:00100000:1.0:1767321588.959927:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718656 00000100:00100000:1.0:1767321588.959933:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718912 00000100:00100000:1.0:1767321588.959935:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718912 00000100:00100000:1.0:1767321588.959957:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5912 00000100:00100000:1.0:1767321588.959964:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985718784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321588.960001:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985718784:12345-0@lo:400:kworker.0 Request processed in 38us (1075us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321588.960008:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5912 00000100:00100000:1.0:1767321588.960034:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5913 00000100:00100000:1.0:1767321588.960040:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985718528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321588.960073:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985718528:12345-0@lo:400:kworker.0 Request processed in 33us (491us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321588.960079:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5913 00000100:00100000:1.0:1767321588.960090:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6822 00000100:00100000:1.0:1767321588.960093:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985718272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321588.960110:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321588.960138:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bdaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985718272:12345-0@lo:38:kworker.0 Request processed in 43us (430us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321588.960143:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6822 00000100:00100000:1.0:1767321588.960155:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6823 00000100:00100000:1.0:1767321588.960158:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985718656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321588.960192:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321588.960215:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bd9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985718656:12345-0@lo:38:kworker.0 Request processed in 55us (412us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321588.960220:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6823 00000100:00100000:1.0:1767321588.960231:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6824 00000100:00100000:1.0:1767321588.960234:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985718912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321588.960242:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321588.960262:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7012:x1853163985718912:12345-0@lo:38:kworker.0 Request processed in 26us (428us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321588.960267:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6824 00000100:00100000:1.0:1767321588.960285:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985718400 00000100:00100000:1.0:1767321588.960289:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985718400 00000100:00100000:1.0:1767321588.960309:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2196 00000100:00100000:1.0:1767321588.960313:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985718400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321588.960322:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321588.960346:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b5bda680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985718400:12345-0@lo:38:kworker.0 Request processed in 32us (584us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321588.960352:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2196 00000100:00100000:1.0:1767321588.960388:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985718528:0@lo:400:kworker.0 00000100:00080000:1.0:1767321588.960417:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321588.960422:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321588.960426:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985718272:0@lo:38:kworker.0 00000100:00080000:1.0:1767321588.960432:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321588.960436:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321588.960438:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985718400:0@lo:38:kworker.0 00000100:00080000:1.0:1767321588.960448:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321588.960451:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321588.960454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985718656:0@lo:38:kworker.0 00000100:00080000:1.0:1767321588.960460:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321588.960463:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321588.960466:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985718912:0@lo:38:kworker.0 00000100:00100000:1.0:1767321588.960494:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985718784:0@lo:400:kworker.0 00000100:00080000:3.0:1767321588.960651:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321588.961507:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985719168 00000100:00100000:0.0:1767321588.961515:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985719168 00000100:00100000:0.0:1767321588.961560:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3808 00000100:00100000:0.0:1767321588.961568:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985719168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321588.961647:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1af73ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985719168:12345-0@lo:400:kworker.0 Request processed in 80us (2120us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321588.961654:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3808 00000100:00100000:1.0:1767321588.961906:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985719168:0@lo:400:kworker.0 00010000:00010000:1.0:1767321589.595472:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321590.622812:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321591.643574:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321592.667442:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321593.051286:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321568/real 1767321568] req@ffff8ce1a4df9500 x1853163985713792/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321593 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321593.051317:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a4df9500 x1853163985713792/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321593 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321593.051331:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321593.051429:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321593.051434:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985713792:0@lo:250:kworker.0 00000100:00080000:0.0:1767321593.053630:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321593.053642:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321593.053646:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321593.053654:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6816 00000100:00080000:0.0:1767321593.053659:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321593.053666:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321593.053702:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f4380 x1853163985719296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321593.053737:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321593.053741:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321593.053743:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321593.053746:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6837 00000100:00080000:0.0:1767321593.053749:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321593.053752:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321593.053767:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7480 x1853163985719424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321593.053777:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321593.053780:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321593.053782:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321593.053785:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6837 00000100:00080000:0.0:1767321593.053787:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321593.053790:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321593.053802:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f6d80 x1853163985719552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321593.053846:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321593.053850:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321593.053852:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321593.053857:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6837 00000100:00080000:0.0:1767321593.053859:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321593.053862:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321593.053883:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188631180 x1853163985719808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321593.053916:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321593.053919:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321593.053921:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321593.053924:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6837 00000100:00100000:3.0:1767321593.053924:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1767321593.053926:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321593.053929:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321593.053949:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985719680:0@lo:400:kworker.0 00000100:00100000:0.0:1767321593.053952:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce188633b80 x1853163985720064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321593.053988:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985719936:0@lo:400:kworker.0 00000100:00100000:3.0:1767321593.054014:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321593.054051:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985719680 00000100:00100000:0.0:1767321593.054059:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985719680 00000100:00100000:3.0:1767321593.054091:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985720320:0@lo:400:kworker.0 00000100:00100000:0.0:1767321593.054110:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5914 00000100:00100000:3.0:1767321593.054116:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321593.054119:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985719680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321593.054148:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720320 00000100:00100000:3.0:1767321593.054151:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720320 00000100:00100000:0.0:1767321593.054182:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163985719680:12345-0@lo:400:kworker.0 Request processed in 65us (167us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321593.054190:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5914 00000100:00100000:3.0:1767321593.054207:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3809 00000100:00100000:3.0:1767321593.054215:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985720320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321593.054224:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321593.054244:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321593.054244:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a11500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985720320:12345-0@lo:400:kworker.0 Request processed in 31us (127us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321593.054249:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985720192:0@lo:400:kworker.0 00000100:00100000:3.0:1767321593.054250:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3809 00000100:00100000:0.0:1767321593.054269:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321593.054285:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985719936 00000100:00100000:0.0:1767321593.054287:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985719936 00000100:00100000:3.0:1767321593.054296:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985720320:0@lo:400:kworker.0 00000100:00100000:0.0:1767321593.054308:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5915 00000100:00100000:3.0:1767321593.054312:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163985719680:0@lo:400:kworker.0 00000100:00100000:0.0:1767321593.054314:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985719936:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321593.054337:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720192 00000100:00100000:0.0:1767321593.054341:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f05880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985719936:12345-0@lo:400:kworker.0 Request processed in 28us (116us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321593.054341:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720192 00000100:00100000:0.0:1767321593.054346:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5915 00000100:00100000:3.0:1767321593.054368:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3810 00000100:00100000:3.0:1767321593.054374:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985720192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321593.054379:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985719936:0@lo:400:kworker.0 00000100:00080000:0.0:1767321593.054393:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321593.054396:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321593.054398:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321593.054412:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f04700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985720192:12345-0@lo:400:kworker.0 Request processed in 37us (142us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321593.054418:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188f07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985720192:0@lo:400:kworker.0 00000100:00100000:3.0:1767321593.054418:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3810 00000100:00100000:1.0:1767321593.054732:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985719296:0@lo:250:kworker.0 00000400:00000100:1.0:1767321593.054758:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985719296 offset 0 length 520: 4 00000100:00100000:1.0:1767321593.054777:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985719424:0@lo:38:kworker.0 00000100:00100000:1.0:1767321593.054796:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321593.054822:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985719552:0@lo:38:kworker.0 00000100:00100000:1.0:1767321593.054846:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321593.054863:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985719424 00000100:00100000:3.0:1767321593.054867:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985719424 00000100:00100000:1.0:1767321593.054872:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985719808:0@lo:38:kworker.0 00000100:00100000:1.0:1767321593.054892:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321593.054896:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6825 00000100:00100000:3.0:1767321593.054902:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985719424:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321593.054917:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985720064:0@lo:38:kworker.0 00010000:00020000:3.0:1767321593.054918:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321593.054948:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321593.054956:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985719424:12345-0@lo:38:kworker.0 Request processed in 54us (160us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321593.054962:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6825 00000100:00080000:1.0:1767321593.054971:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767321593.054971:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985719808 00000100:00100000:3.0:1767321593.054974:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985719808 00000100:00080000:1.0:1767321593.054975:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321593.054978:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985719424:0@lo:38:kworker.0 00000100:00100000:3.0:1767321593.054988:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720064 00000100:00100000:3.0:1767321593.054990:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720064 00000100:00100000:1.0:1767321593.054999:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6826 00000100:00100000:3.0:1767321593.055003:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6827 00000100:00100000:1.0:1767321593.055004:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985719808:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321593.055007:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985720064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321593.055014:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767321593.055016:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321593.055038:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db77480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985719808:12345-0@lo:38:kworker.0 Request processed in 33us (146us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321593.055044:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6826 00000100:00100000:0.0:1767321593.055192:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985719552 00000100:00100000:0.0:1767321593.055226:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985719552 00000100:00100000:0.0:1767321593.055261:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2197 00000100:00100000:0.0:1767321593.055266:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985719552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321593.055276:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321593.056994:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db75c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985720064:12345-0@lo:38:kworker.0 Request processed in 1977us (2036us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321593.057009:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6827 00000100:00080000:1.0:1767321593.060474:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321593.060488:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321593.060493:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985719808:0@lo:38:kworker.0 00000100:00080000:1.0:1767321593.060532:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321593.060536:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321593.060539:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985720064:0@lo:38:kworker.0 00000100:00100000:0.0:1767321593.060710:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19db74a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985719552:12345-0@lo:38:kworker.0 Request processed in 5439us (5859us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321593.060722:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2197 00000100:00080000:1.0:1767321593.064793:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321593.064806:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321593.064811:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985719552:0@lo:38:kworker.0 00000100:00100000:0.0:1767321593.563505:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321593.563542:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985720448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321593.563613:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321593.563707:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720448 00000100:00100000:3.0:1767321593.563715:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720448 00000100:00100000:3.0:1767321593.563785:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3046 00000100:00100000:3.0:1767321593.563796:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985720448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321593.563829:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321593.563887:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985720448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321593.563896:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3046 00000100:00100000:3.0:1767321593.564300:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321593.564312:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985720576:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321593.564342:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321593.564487:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985720448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321593.564591:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720576 00000100:00100000:1.0:1767321593.564596:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720576 00000100:00100000:1.0:1767321593.564632:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3047 00000100:00100000:1.0:1767321593.564639:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985720576:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321593.564656:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321593.564703:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985720576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 64us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321593.564710:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3047 00000100:00100000:0.0:1767321593.564903:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985720576:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321593.692061:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767321594.028529:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920681984 offset 0 length 520: 4 00000100:00100000:3.0:1767321594.029118:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321594.029158:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321594.029263:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920682112 00000100:00100000:2.0:1767321594.029273:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920682112 00000100:00100000:2.0:1767321594.029312:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920682240 00000100:00100000:2.0:1767321594.029315:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920682240 00000100:00100000:2.0:1767321594.029362:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5916 00000100:00100000:2.0:1767321594.029372:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920682240:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321594.029461:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920682240:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 91us (302us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321594.029468:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5916 00000100:00100000:2.0:1767321594.029493:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6828 00000100:00100000:2.0:1767321594.029497:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920682112:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321594.029517:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321594.029542:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a12680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920682112:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 43us (427us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321594.029548:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6828 00000100:00080000:3.0:1767321594.029850:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321594.716780:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321595.745288:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321596.767327:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321597.794632:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321598.171302:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321598.171313:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321598.171317:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321598.171321:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321598.171325:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321598.171346:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6841 00000100:00080000:1.0:1767321598.171351:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321598.171359:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321598.171400:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76a00 x1853163985720704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321598.171416:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321598.171419:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321598.171421:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321598.171425:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6841 00000100:00080000:1.0:1767321598.171427:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321598.171431:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321598.171444:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db74000 x1853163985720832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321598.171489:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321598.171493:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321598.171495:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321598.171499:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6841 00000100:00080000:1.0:1767321598.171501:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321598.171505:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321598.171518:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7480 x1853163985721088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321598.171527:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985720960:0@lo:400:kworker.0 00000100:00080000:1.0:1767321598.171551:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321598.171555:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321598.171556:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321598.171560:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6841 00000100:00080000:1.0:1767321598.171563:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321598.171567:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321598.171579:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985721344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321598.171597:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321598.171631:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985721216:0@lo:400:kworker.0 00000100:00100000:0.0:1767321598.171669:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321598.171669:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321598.171677:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985721600:0@lo:400:kworker.0 00000100:00100000:1.0:1767321598.171699:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321598.171707:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985721472:0@lo:400:kworker.0 00000100:00100000:0.0:1767321598.171714:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321598.171746:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321598.171814:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985720704:0@lo:38:kworker.0 00000100:00100000:1.0:1767321598.171872:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720960 00000100:00100000:1.0:1767321598.171879:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720960 00000100:00100000:1.0:1767321598.171932:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721216 00000100:00100000:1.0:1767321598.171936:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721216 00000100:00100000:1.0:1767321598.171959:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5917 00000100:00100000:1.0:1767321598.171967:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985720960:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321598.172079:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321598.172091:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721600 00000100:00100000:3.0:1767321598.172094:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721472 00000100:00100000:0.0:1767321598.172097:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721600 00000100:00100000:3.0:1767321598.172100:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721472 00000100:00100000:0.0:1767321598.172144:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3811 00000100:00100000:3.0:1767321598.172155:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3812 00000100:00100000:0.0:1767321598.172156:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985721600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321598.172200:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985721472:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321598.172447:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321598.172477:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985720832:0@lo:38:kworker.0 00000100:00100000:2.0:1767321598.172502:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321598.172527:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985721088:0@lo:38:kworker.0 00000100:00100000:0.0:1767321598.172558:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f06300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985721600:12345-0@lo:400:kworker.0 Request processed in 404us (846us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321598.172564:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321598.172572:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3811 00000100:00100000:2.0:1767321598.172592:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985721344:0@lo:38:kworker.0 00000100:00100000:0.0:1767321598.172608:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720704 00000100:00100000:0.0:1767321598.172614:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720704 00000100:00100000:2.0:1767321598.172618:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321598.172645:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721088 00000100:00100000:2.0:1767321598.172649:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721088 00000100:00100000:2.0:1767321598.172664:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721344 00000100:00100000:2.0:1767321598.172667:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721344 00000100:00100000:2.0:1767321598.172675:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5918 00000100:00100000:2.0:1767321598.172697:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985721216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321598.172736:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163985721216:12345-0@lo:400:kworker.0 Request processed in 43us (1068us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321598.172743:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5918 00000100:00100000:2.0:1767321598.172757:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6829 00000100:00100000:2.0:1767321598.172760:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985720704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321598.172779:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321598.172812:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985720704:12345-0@lo:38:kworker.0 Request processed in 49us (364us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321598.172817:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6829 00000100:00100000:2.0:1767321598.172829:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6830 00000100:00100000:2.0:1767321598.172833:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985721088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321598.172851:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321598.172876:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985721088:12345-0@lo:38:kworker.0 Request processed in 41us (315us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321598.172881:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6830 00000100:00100000:2.0:1767321598.172894:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6831 00000100:00100000:2.0:1767321598.172897:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985721344:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321598.172904:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321598.172924:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985721344:12345-0@lo:38:kworker.0 Request processed in 25us (309us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321598.172929:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6831 00000100:00100000:2.0:1767321598.172966:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163985721216:0@lo:400:kworker.0 00000100:00080000:2.0:1767321598.172999:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321598.173003:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321598.173007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985720704:0@lo:38:kworker.0 00000100:00080000:2.0:1767321598.173014:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321598.173017:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321598.173020:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985721088:0@lo:38:kworker.0 00000100:00080000:2.0:1767321598.173027:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321598.173029:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321598.173032:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985721344:0@lo:38:kworker.0 00000100:00080000:2.0:1767321598.173082:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321598.173086:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321598.173096:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985720832 00000100:00100000:2.0:1767321598.173099:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985720832 00000100:00100000:2.0:1767321598.173115:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2198 00000100:00100000:2.0:1767321598.173118:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985720832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321598.173124:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321598.173148:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61c6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985720832:12345-0@lo:38:kworker.0 Request processed in 29us (646us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321598.173154:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2198 00000100:00080000:2.0:1767321598.173200:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321598.173204:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321598.173207:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985720832:0@lo:38:kworker.0 00000100:00100000:2.0:1767321598.173251:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985721600:0@lo:400:kworker.0 00000100:00100000:1.0:1767321598.173359:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985720960:12345-0@lo:400:kworker.0 Request processed in 1391us (1761us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321598.173364:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5917 00000100:00100000:2.0:1767321598.173929:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985720960:0@lo:400:kworker.0 00000100:00100000:3.0:1767321598.174211:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985721472:12345-0@lo:400:kworker.0 Request processed in 2045us (2463us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321598.174220:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3812 00000100:00100000:2.0:1767321598.174286:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985721472:0@lo:400:kworker.0 00000100:00100000:0.0:1767321598.683711:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321598.683735:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985721728:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321598.683805:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321598.683930:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985721856:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321598.684666:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321598.690908:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721728 00000100:00100000:1.0:1767321598.690923:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721728 00000100:00100000:1.0:1767321598.694874:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721856 00000100:00100000:1.0:1767321598.694892:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721856 00000100:00100000:0.0:1767321598.694958:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3048 00000100:00100000:0.0:1767321598.694971:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985721728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321598.695006:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321598.695132:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188f07800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985721728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 163us (11326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321598.695141:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3048 00000100:00100000:0.0:1767321598.695159:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3049 00000100:00100000:0.0:1767321598.695184:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985721856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321598.695199:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321598.695229:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985721856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (10565us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321598.695236:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3049 00000100:00100000:3.0:1767321598.695566:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce184a13100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985721728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321598.695601:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985721856:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321598.811433:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321599.154704:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321599.154858:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321599.159272:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920682496 00000100:00100000:0.0:1767321599.159295:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920682496 00000100:00080000:2.0:1767321599.159416:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321599.164024:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920682368 00000100:00100000:1.0:1767321599.164048:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920682368 00000100:00100000:1.0:1767321599.164410:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5919 00000100:00100000:1.0:1767321599.164434:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920682496:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321599.164571:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920682496:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 150us (9886us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321599.164581:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5919 00000100:00100000:1.0:1767321599.164609:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6832 00000100:00100000:1.0:1767321599.164614:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920682368:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321599.164640:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321599.164670:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a10e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920682368:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (9812us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321599.164682:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6832 00010000:00010000:2.0:1767321599.836458:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321600.860430:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321601.883554:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321602.907628:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321603.291692:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321603.291707:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321603.291711:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321603.291716:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321603.291721:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321603.291732:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6846 00000100:00080000:2.0:1767321603.291738:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321603.291756:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321603.291811:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21880 x1853163985721984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321603.291841:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321603.291845:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321603.291847:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321603.291851:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6846 00000100:00080000:2.0:1767321603.291854:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321603.291858:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321603.291872:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985722112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321603.292271:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985721984:0@lo:38:kworker.0 00000100:00100000:0.0:1767321603.292379:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321603.292478:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985722112:0@lo:38:kworker.0 00000100:00100000:0.0:1767321603.292508:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321603.292628:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985721984 00000100:00100000:3.0:1767321603.292639:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985721984 00000100:00100000:3.0:1767321603.292697:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6833 00000100:00100000:3.0:1767321603.292707:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985721984:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321603.292736:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321603.292806:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985721984:12345-0@lo:38:kworker.0 Request processed in 99us (426us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321603.292814:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6833 00000100:00080000:3.0:1767321603.292863:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321603.292869:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321603.292873:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985721984:0@lo:38:kworker.0 00000100:00100000:1.0:1767321603.293185:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985722112 00000100:00100000:1.0:1767321603.293190:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985722112 00000100:00100000:1.0:1767321603.293232:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2199 00000100:00100000:1.0:1767321603.293238:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985722112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321603.293249:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321603.293299:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5939f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985722112:12345-0@lo:38:kworker.0 Request processed in 61us (790us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321603.293305:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2199 00000100:00080000:3.0:1767321603.293426:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321603.293431:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321603.293434:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985722112:0@lo:38:kworker.0 00000100:00100000:3.0:1767321603.293516:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985722240:0@lo:400:kworker.0 00000100:00100000:3.0:1767321603.293553:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321603.294522:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321603.294531:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321603.294536:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321603.294545:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6846 00000100:00080000:2.0:1767321603.294550:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321603.294560:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321603.294598:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe22680 x1853163985722368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321603.294656:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321603.294660:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321603.294662:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321603.294666:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6846 00000100:00080000:2.0:1767321603.294669:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321603.294673:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321603.294684:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1bbe21c00 x1853163985722624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321603.294696:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321603.294717:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985722752:0@lo:400:kworker.0 00000100:00100000:1.0:1767321603.294784:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985722240 00000100:00100000:1.0:1767321603.294793:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985722240 00000100:00100000:1.0:1767321603.294863:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5920 00000100:00100000:1.0:1767321603.294875:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985722240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295000:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321603.295031:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985722496:0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295055:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321603.295088:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985722752 00000100:00100000:0.0:1767321603.295092:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985722752 00000100:00100000:0.0:1767321603.295128:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3813 00000100:00100000:0.0:1767321603.295134:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985722752:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295189:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985722752:12345-0@lo:400:kworker.0 Request processed in 53us (187us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321603.295196:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3813 00000100:00100000:0.0:1767321603.295222:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985722880:0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295245:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321603.295271:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985722880 00000100:00100000:0.0:1767321603.295275:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985722880 00000100:00100000:0.0:1767321603.295311:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3814 00000100:00100000:0.0:1767321603.295317:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985722880:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295350:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985722880:12345-0@lo:400:kworker.0 Request processed in 33us (103us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321603.295356:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3814 00000100:00100000:0.0:1767321603.295394:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985722880:0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295422:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985722752:0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295440:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985722368:0@lo:38:kworker.0 00000100:00100000:0.0:1767321603.295468:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321603.295501:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985722624:0@lo:38:kworker.0 00000100:00100000:0.0:1767321603.295523:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321603.295555:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985722496 00000100:00100000:0.0:1767321603.295558:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985722496 00000100:00100000:0.0:1767321603.295583:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985722368 00000100:00100000:0.0:1767321603.295586:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985722368 00000100:00100000:0.0:1767321603.295595:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985722624 00000100:00100000:0.0:1767321603.295597:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985722624 00000100:00100000:0.0:1767321603.295606:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5921 00000100:00100000:0.0:1767321603.295613:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985722496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.295648:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a5938a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7016:x1853163985722496:12345-0@lo:400:kworker.0 Request processed in 35us (592us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321603.295654:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5921 00000100:00100000:0.0:1767321603.295665:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6834 00000100:00100000:0.0:1767321603.295669:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985722368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321603.295689:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321603.295720:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985722368:12345-0@lo:38:kworker.0 Request processed in 50us (251us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321603.295726:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6834 00000100:00100000:0.0:1767321603.295738:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6835 00000100:00100000:0.0:1767321603.295742:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985722624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321603.295768:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321603.295791:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985722624:12345-0@lo:38:kworker.0 Request processed in 47us (268us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321603.295797:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6835 00000100:00100000:0.0:1767321603.295823:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7016:1853163985722496:0@lo:400:kworker.0 00000100:00080000:0.0:1767321603.295845:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321603.295849:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321603.295853:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985722368:0@lo:38:kworker.0 00000100:00080000:0.0:1767321603.295861:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321603.295864:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321603.295867:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985722624:0@lo:38:kworker.0 00000100:00080000:0.0:1767321603.295907:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321603.295912:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321603.295914:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321603.302175:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce184a13480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985722240:12345-0@lo:400:kworker.0 Request processed in 7277us (8595us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321603.302194:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5920 00000100:00100000:3.0:1767321603.304017:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1bbe23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985722240:0@lo:400:kworker.0 00000100:00100000:0.0:1767321603.804966:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321603.804997:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985723008:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321603.805087:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321603.805255:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723008 00000100:00100000:3.0:1767321603.805267:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723008 00000100:00100000:3.0:1767321603.805346:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3050 00000100:00100000:3.0:1767321603.805357:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985723008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321603.805397:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321603.805955:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985723008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 596us (863us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321603.805966:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3050 00000100:00100000:0.0:1767321603.806112:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985723008:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321603.807621:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321603.807644:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985723136:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321603.807718:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321603.807793:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723136 00000100:00100000:3.0:1767321603.807803:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723136 00000100:00100000:3.0:1767321603.807875:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3051 00000100:00100000:3.0:1767321603.807886:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985723136:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321603.807923:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321603.807988:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985723136:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (269us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321603.808007:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3051 00000100:00100000:2.0:1767321603.808099:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b34d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985723136:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767321603.931523:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321604.270047:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321604.270104:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321604.270216:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920682752 00000100:00100000:2.0:1767321604.270231:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920682752 00000100:00100000:2.0:1767321604.270335:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920682624 00000100:00100000:2.0:1767321604.270339:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920682624 00000100:00100000:2.0:1767321604.270375:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5922 00000100:00100000:2.0:1767321604.270391:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920682752:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:2.0:1767321604.270489:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2514:x1853163920682752:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 103us (456us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321604.270498:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5922 00000100:00100000:2.0:1767321604.270521:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6836 00000100:00100000:2.0:1767321604.270527:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920682624:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321604.270552:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321604.270583:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3ff5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2512:x1853163920682624:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 54us (477us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321604.270590:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6836 00000100:00080000:0.0:1767321604.271096:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321604.955883:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321605.979573:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321607.004104:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321608.027386:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321608.412428:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321608.412442:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321608.412445:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321608.412450:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321608.412456:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321608.412476:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6851 00000100:00080000:2.0:1767321608.412483:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321608.412493:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321608.412554:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524ce00 x1853163985723264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321608.412589:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321608.412593:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321608.412595:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321608.412599:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6851 00000100:00080000:2.0:1767321608.412602:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321608.412606:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321608.412631:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b524fb80 x1853163985723392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321608.412704:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321608.412709:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321608.412710:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321608.412714:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6851 00000100:00080000:2.0:1767321608.412717:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321608.412720:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321608.412733:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a61f7480 x1853163985723648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321608.412763:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321608.412766:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321608.412768:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321608.412772:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6851 00000100:00080000:2.0:1767321608.412774:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321608.412778:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321608.412789:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76a00 x1853163985723904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321608.412840:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985724032:0@lo:400:kworker.0 00000100:00100000:2.0:1767321608.412932:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321608.412994:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321608.412998:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985723776:0@lo:400:kworker.0 00000100:00100000:2.0:1767321608.413017:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321608.413393:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985724160:0@lo:400:kworker.0 00000100:00100000:1.0:1767321608.413467:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321608.413524:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321608.413531:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985723520:0@lo:400:kworker.0 00000100:00100000:1.0:1767321608.413554:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321608.413609:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723776 00000100:00100000:1.0:1767321608.413616:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723776 00000100:00080000:3.0:1767321608.413671:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321608.413698:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723520 00000100:00100000:1.0:1767321608.413704:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723520 00000100:00100000:3.0:1767321608.413734:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5923 00000100:00100000:2.0:1767321608.413791:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724032 00000100:00100000:2.0:1767321608.413798:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724032 00000100:00100000:1.0:1767321608.413828:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5924 00000100:00100000:1.0:1767321608.413843:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985723520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321608.413856:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724160 00000100:00100000:2.0:1767321608.413861:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724160 00000100:00100000:2.0:1767321608.413881:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3815 00000100:00100000:2.0:1767321608.413890:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985724032:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321608.413954:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985724032:12345-0@lo:400:kworker.0 Request processed in 66us (1026us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321608.413960:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3815 00000100:00100000:2.0:1767321608.413974:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3816 00000100:00100000:2.0:1767321608.413978:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985724160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321608.414000:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985724160:12345-0@lo:400:kworker.0 Request processed in 22us (535us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321608.414003:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3816 00000100:00100000:2.0:1767321608.414042:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985724032:0@lo:400:kworker.0 00000100:00080000:2.0:1767321608.414074:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321608.414077:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321608.414080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321608.414204:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985723776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321608.414252:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985723776:12345-0@lo:400:kworker.0 Request processed in 53us (1233us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321608.414257:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5923 00000100:00100000:3.0:1767321608.414292:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985723520:0@lo:400:kworker.0 00000100:00100000:3.0:1767321608.414300:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985724160:0@lo:400:kworker.0 00000100:00100000:1.0:1767321608.414488:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985723520:12345-0@lo:400:kworker.0 Request processed in 82us (364us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321608.414499:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5924 00000100:00100000:0.0:1767321608.414520:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985723264:0@lo:38:kworker.0 00000100:00100000:0.0:1767321608.414572:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321608.414612:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723264 00000100:00100000:1.0:1767321608.414620:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723264 00000100:00100000:0.0:1767321608.414635:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985723392:0@lo:38:kworker.0 00000100:00100000:1.0:1767321608.414676:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6837 00000100:00100000:0.0:1767321608.414684:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321608.414692:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985723264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321608.414718:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321608.414727:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985723648:0@lo:38:kworker.0 00000100:00100000:0.0:1767321608.414771:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321608.414784:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985723264:12345-0@lo:38:kworker.0 Request processed in 94us (212us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321608.414795:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6837 00000100:00100000:0.0:1767321608.414800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985723904:0@lo:38:kworker.0 00000100:00100000:1.0:1767321608.414812:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723648 00000100:00100000:1.0:1767321608.414820:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723648 00000100:00100000:0.0:1767321608.414825:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321608.414852:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723904 00000100:00080000:0.0:1767321608.414857:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767321608.414860:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723904 00000100:00080000:0.0:1767321608.414863:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321608.414869:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985723264:0@lo:38:kworker.0 00000100:00100000:1.0:1767321608.414900:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6838 00000100:00100000:0.0:1767321608.414902:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985723392 00000100:00100000:0.0:1767321608.414907:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985723392 00000100:00100000:1.0:1767321608.414909:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985723648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321608.414933:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321608.414947:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2200 00000100:00100000:0.0:1767321608.414953:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985723392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321608.414967:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321608.415474:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985723776:0@lo:400:kworker.0 00000100:00100000:2.0:1767321608.415518:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6839 00000100:00100000:2.0:1767321608.415523:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985723904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321608.415532:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321608.415629:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985723392:12345-0@lo:38:kworker.0 Request processed in 659us (933us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321608.415638:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2200 00000100:00080000:0.0:1767321608.415696:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321608.415704:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321608.415707:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b524fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985723392:0@lo:38:kworker.0 00000100:00080000:0.0:1767321608.415719:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321608.415722:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321608.415725:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985723648:0@lo:38:kworker.0 00000100:00100000:1.0:1767321608.416861:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe22680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985723648:12345-0@lo:38:kworker.0 Request processed in 1941us (2079us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321608.416877:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6838 00000100:00100000:2.0:1767321608.416912:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe20000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985723904:12345-0@lo:38:kworker.0 Request processed in 1381us (2079us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321608.416924:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6839 00000100:00080000:0.0:1767321608.417048:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321608.417055:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321608.417059:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985723904:0@lo:38:kworker.0 00000100:00100000:2.0:1767321608.924406:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985724288:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321608.924516:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321608.924730:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724288 00000100:00100000:0.0:1767321608.924741:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724288 00000100:00100000:0.0:1767321608.924809:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3052 00000100:00100000:0.0:1767321608.924816:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985724288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321608.924853:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321608.924921:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985724288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (404us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321608.924937:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3052 00000100:00100000:0.0:1767321608.925137:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985724416:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321608.925184:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321608.925249:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724416 00000100:00100000:3.0:1767321608.925253:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724416 00000100:00100000:3.0:1767321608.925279:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3053 00000100:00100000:3.0:1767321608.925285:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985724416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321608.925309:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321608.925342:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1bbe21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985724416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (157us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321608.925346:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3053 00000100:00100000:0.0:1767321608.925492:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985724416:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321608.925692:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188631180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985724288:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321609.051528:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321609.392407:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321609.392520:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321609.392753:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920683008 00000100:00100000:0.0:1767321609.392767:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920683008 00000100:00100000:0.0:1767321609.392854:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920682880 00000100:00100000:0.0:1767321609.392859:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920682880 00000100:00100000:0.0:1767321609.392883:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5925 00000100:00100000:0.0:1767321609.392895:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920683008:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321609.393009:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920683008:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 116us (617us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321609.393018:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5925 00000100:00100000:0.0:1767321609.393044:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6840 00000100:00100000:0.0:1767321609.393048:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920682880:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321609.393075:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321609.393105:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188631180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920682880:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 55us (585us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321609.393112:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6840 00000100:00080000:3.0:1767321609.394065:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767321610.078023:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321611.099958:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321612.123480:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321613.147851:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321613.531359:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321613.531374:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321613.531378:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321613.531383:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321613.531390:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321613.531400:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6856 00000100:00080000:2.0:1767321613.531406:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321613.531416:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321613.531476:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dca4380 x1853163985724544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321613.531496:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321613.531499:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321613.531501:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321613.531504:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6856 00000100:00080000:2.0:1767321613.531507:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321613.531510:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321613.531524:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19db76a00 x1853163985724672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321613.531576:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321613.531580:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321613.531582:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321613.531586:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6856 00000100:00080000:2.0:1767321613.531588:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321613.531592:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321613.531608:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a4df9500 x1853163985724928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321613.531636:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985724800:0@lo:400:kworker.0 00000100:00080000:2.0:1767321613.531648:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321613.531654:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321613.531658:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767321613.531659:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985725056:0@lo:400:kworker.0 00000100:00080000:2.0:1767321613.531663:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6856 00000100:00080000:2.0:1767321613.531669:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321613.531674:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321613.531688:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163985725184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321613.531737:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321613.531755:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985725312:0@lo:400:kworker.0 00000100:00100000:1.0:1767321613.531782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985724544:0@lo:38:kworker.0 00000100:00100000:3.0:1767321613.531807:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321613.531830:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321613.531841:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725312 00000100:00100000:0.0:1767321613.531854:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725312 00000100:00100000:3.0:1767321613.531884:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725056 00000100:00100000:3.0:1767321613.531889:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725056 00000100:00100000:1.0:1767321613.532058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985724672:0@lo:38:kworker.0 00000100:00100000:3.0:1767321613.532197:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724544 00000100:00100000:3.0:1767321613.532201:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724544 00000100:00100000:0.0:1767321613.532208:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3817 00000100:00100000:1.0:1767321613.532220:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321613.532226:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985725312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321613.532227:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5926 00000100:00100000:3.0:1767321613.532242:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985725056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321613.532260:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985724928:0@lo:38:kworker.0 00000100:00100000:1.0:1767321613.532295:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321613.532306:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985725312:12345-0@lo:400:kworker.0 Request processed in 87us (498us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321613.532318:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3817 00000100:00100000:1.0:1767321613.532320:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985725184:0@lo:38:kworker.0 00000100:00100000:3.0:1767321613.532345:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985725056:12345-0@lo:400:kworker.0 Request processed in 107us (610us total) trans 0 rc 0/0 00000100:00080000:0.0:1767321613.532355:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321613.532357:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5926 00000100:00100000:1.0:1767321613.532360:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321613.532364:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321613.532382:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724928 00000100:00100000:3.0:1767321613.532384:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725184 00000100:00100000:0.0:1767321613.532387:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724928 00000100:00100000:3.0:1767321613.532388:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725184 00000100:00100000:3.0:1767321613.532405:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6841 00000100:00100000:3.0:1767321613.532410:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985724544:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321613.532418:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6842 00000100:00100000:0.0:1767321613.532426:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985725184:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321613.532431:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767321613.532442:0:15468:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321613.532443:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6843 00000100:00100000:2.0:1767321613.532457:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985724928:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321613.532470:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321613.532637:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7012:x1853163985725184:12345-0@lo:38:kworker.0 Request processed in 210us (282us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321613.532644:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6842 00000100:00100000:3.0:1767321613.532676:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985724544:12345-0@lo:38:kworker.0 Request processed in 265us (847us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321613.532684:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321613.532684:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6841 00000100:00100000:3.0:1767321613.532702:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724800 00000100:00100000:3.0:1767321613.532705:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724800 00000100:00100000:0.0:1767321613.532717:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321613.532729:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985725440:0@lo:400:kworker.0 00000100:00100000:3.0:1767321613.532732:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5927 00000100:00100000:3.0:1767321613.532739:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985724800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321613.532844:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321613.532848:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b524ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985724800:12345-0@lo:400:kworker.0 Request processed in 107us (167us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321613.532853:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5927 00000100:00100000:0.0:1767321613.532892:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985724800:0@lo:400:kworker.0 00000100:00100000:0.0:1767321613.532925:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725440 00000100:00080000:3.0:1767321613.532928:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321613.532929:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725440 00000100:00100000:3.0:1767321613.532949:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985725056:0@lo:400:kworker.0 00000100:00100000:0.0:1767321613.532974:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3818 00000100:00100000:0.0:1767321613.532985:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985725440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321613.532988:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985725312:0@lo:400:kworker.0 00000100:00080000:3.0:1767321613.533017:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321613.533040:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985725440:12345-0@lo:400:kworker.0 Request processed in 56us (196us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321613.533048:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3818 00000100:00100000:0.0:1767321613.533078:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985725440:0@lo:400:kworker.0 00000100:00100000:0.0:1767321613.533108:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985724672 00000100:00100000:0.0:1767321613.533111:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985724672 00000100:00100000:0.0:1767321613.533141:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2201 00000100:00100000:0.0:1767321613.533146:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985724672:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321613.533160:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321613.533224:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985724672:12345-0@lo:38:kworker.0 Request processed in 76us (1007us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321613.533231:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2201 00000100:00100000:2.0:1767321613.536014:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985724928:12345-0@lo:38:kworker.0 Request processed in 3545us (3703us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321613.536032:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6843 00000100:00080000:1.0:1767321613.537378:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321613.537391:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321613.537408:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985725184:0@lo:38:kworker.0 00000100:00080000:1.0:1767321613.537450:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321613.537455:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321613.537458:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dca4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985724544:0@lo:38:kworker.0 00000100:00080000:1.0:1767321613.537471:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321613.537474:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321613.537477:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19db76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985724672:0@lo:38:kworker.0 00000100:00080000:1.0:1767321613.537486:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321613.537490:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321613.537493:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a4df9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985724928:0@lo:38:kworker.0 00000100:00100000:1.0:1767321614.043962:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985725568:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321614.044066:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321614.044208:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725568 00000100:00100000:3.0:1767321614.044219:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725568 00000100:00100000:3.0:1767321614.044299:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3054 00000100:00100000:3.0:1767321614.044312:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985725568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321614.044368:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321614.044431:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985725568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 122us (364us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321614.044439:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3054 00000100:00100000:3.0:1767321614.044499:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985725568:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321614.044834:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985725696:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321614.044869:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321614.044938:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725696 00000100:00100000:2.0:1767321614.044944:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725696 00000100:00100000:2.0:1767321614.044978:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3055 00000100:00100000:2.0:1767321614.044986:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985725696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321614.045007:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:1.0:1767321614.052872:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce188633b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985725696:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321614.053352:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985725696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 8360us (8476us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321614.053367:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3055 00010000:00010000:1.0:1767321614.171589:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321614.507507:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321614.508610:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920683136 00000100:00100000:2.0:1767321614.508625:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920683136 00000100:00100000:2.0:1767321614.508700:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6844 00000100:00100000:2.0:1767321614.508710:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920683136:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321614.508740:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321614.508851:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce188633b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:2512:x1853163920683136:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 140us (1356us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321614.508860:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6844 00000100:00100000:0.0:1767321614.515954:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321614.516045:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920683264 00000100:00100000:3.0:1767321614.516055:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920683264 00000100:00100000:3.0:1767321614.516130:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5928 00000100:00100000:3.0:1767321614.516140:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920683264:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:3.0:1767321614.516253:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920683264:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 114us (308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321614.516260:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5928 00000100:00080000:0.0:1767321614.516334:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321615.199326:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321616.219539:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321617.247252:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321618.271722:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321618.602040:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000400:00000100:3.0:1767321618.602085:0:7008:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.26@tcp portal 26 match 1853163920683392 offset 0 length 520: 4 00000100:00100000:3.0:1767321618.602110:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:1.0:1767321618.602357:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920683648 00000100:00100000:1.0:1767321618.602372:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920683648 00000100:00100000:2.0:1767321618.602450:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920683520 00000100:00100000:2.0:1767321618.602490:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920683520 00000100:00100000:1.0:1767321618.602491:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5929 00000100:00100000:1.0:1767321618.602509:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920683648:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:1.0:1767321618.602613:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920683648:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 110us (592us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321618.602622:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5929 00000100:00100000:1.0:1767321618.602646:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6845 00000100:00100000:1.0:1767321618.602651:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920683520:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:1.0:1767321618.602676:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321618.602705:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a61f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920683520:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (594us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321618.602711:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6845 00000100:00080000:0.0:1767321618.602795:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321618.652219:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321618.652229:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321618.652239:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321618.652243:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321618.652248:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321618.652255:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6861 00000100:00080000:2.0:1767321618.652260:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321618.652267:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321618.652302:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a62300 x1853163985725824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321618.652335:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321618.652338:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321618.652340:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321618.652343:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6861 00000100:00080000:2.0:1767321618.652346:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321618.652349:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321618.652361:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60000 x1853163985725952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321618.652384:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321618.652387:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321618.652389:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321618.652392:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6861 00000100:00080000:2.0:1767321618.652394:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321618.652397:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321618.652404:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a6a60700 x1853163985726208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321618.652427:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321618.652429:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321618.652430:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321618.652433:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6861 00000100:00080000:2.0:1767321618.652435:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321618.652438:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321618.652446:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19e9c4e00 x1853163985726464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321618.652505:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985726592:0@lo:400:kworker.0 00000100:00100000:2.0:1767321618.652575:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321618.652631:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321618.652636:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985726336:0@lo:400:kworker.0 00000100:00100000:2.0:1767321618.652659:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321618.652691:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985726336 00000100:00100000:2.0:1767321618.652697:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985726336 00000100:00100000:2.0:1767321618.652743:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5930 00000100:00100000:2.0:1767321618.652750:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985726336:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321618.652792:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985726336:12345-0@lo:400:kworker.0 Request processed in 42us (132us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321618.652799:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5930 00000100:00100000:2.0:1767321618.652841:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985726336:0@lo:400:kworker.0 00000100:00100000:0.0:1767321618.655611:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985726080:0@lo:400:kworker.0 00000100:00100000:0.0:1767321618.655716:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321618.655805:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321618.655814:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985726720:0@lo:400:kworker.0 00000100:00100000:0.0:1767321618.655841:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321618.655880:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321618.655901:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985726592 00000100:00100000:0.0:1767321618.655911:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985726592 00000100:00100000:0.0:1767321618.655978:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985726720 00000100:00100000:0.0:1767321618.655982:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985726720 00000100:00100000:0.0:1767321618.656012:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3819 00000100:00100000:0.0:1767321618.656021:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985726592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321618.656085:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985726592:12345-0@lo:400:kworker.0 Request processed in 64us (3509us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321618.656092:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3819 00000100:00100000:0.0:1767321618.656112:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3820 00000100:00100000:0.0:1767321618.656119:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985726720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321618.656153:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985726720:12345-0@lo:400:kworker.0 Request processed in 36us (311us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321618.656159:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3820 00000100:00080000:0.0:1767321618.656246:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321618.656249:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321618.656284:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985726720:0@lo:400:kworker.0 00000100:00100000:3.0:1767321618.656493:0:7012:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321593/real 1767321593] req@ffff8ce1a61f4380 x1853163985719296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321618 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321618.656517:0:7012:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8ce1a61f4380 x1853163985719296/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321618 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321618.656533:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985725824:0@lo:38:kworker.0 00000100:00100000:3.0:1767321618.656569:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321618.656608:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985725952:0@lo:38:kworker.0 00000100:00100000:3.0:1767321618.656634:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321618.656659:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985726208:0@lo:38:kworker.0 00000100:00100000:3.0:1767321618.656685:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321618.656703:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985726464:0@lo:38:kworker.0 00000100:00100000:3.0:1767321618.656726:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321618.656745:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321618.656761:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321618.656764:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985719296:0@lo:250:kworker.0 00000100:00100000:3.0:1767321618.656794:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985726080 00000100:00100000:3.0:1767321618.656798:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985726080 00000100:00100000:3.0:1767321618.656835:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725824 00000100:00100000:3.0:1767321618.656838:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725824 00000100:00100000:3.0:1767321618.656852:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985726208 00000100:00100000:3.0:1767321618.656855:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985726208 00000100:00100000:3.0:1767321618.656863:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985726464 00000100:00100000:3.0:1767321618.656865:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985726464 00000100:00100000:3.0:1767321618.656875:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5931 00000100:00100000:3.0:1767321618.656882:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985726080:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321618.656918:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985726080:12345-0@lo:400:kworker.0 Request processed in 37us (1203us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321618.656925:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5931 00000100:00100000:3.0:1767321618.656941:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6846 00000100:00100000:3.0:1767321618.656945:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985725824:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321618.656965:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321618.656997:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985725824:12345-0@lo:38:kworker.0 Request processed in 50us (428us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321618.657003:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6846 00000100:00100000:3.0:1767321618.657015:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6847 00000100:00100000:3.0:1767321618.657018:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985726208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321618.657027:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321618.657047:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985726208:12345-0@lo:38:kworker.0 Request processed in 28us (362us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321618.657052:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6847 00000100:00100000:3.0:1767321618.657063:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6848 00000100:00100000:3.0:1767321618.657066:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985726464:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321618.657075:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321618.657095:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985726464:12345-0@lo:38:kworker.0 Request processed in 27us (368us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321618.657101:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6848 00000100:00080000:3.0:1767321618.657123:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321618.657135:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985725952 00000100:00100000:3.0:1767321618.657139:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985725952 00000100:00100000:3.0:1767321618.657160:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2202 00000100:00100000:3.0:1767321618.657194:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985725952:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321618.657204:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321618.657230:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985725952:12345-0@lo:38:kworker.0 Request processed in 35us (594us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321618.657235:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2202 00000100:00080000:3.0:1767321618.657257:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321618.657261:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321618.657265:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce190885000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321618.657269:0:66455:0:(import.c:509:import_select_connection()) MGC192.168.204.126@tcp: connect to NID 0@lo last attempt 6841 00000100:00080000:3.0:1767321618.657274:0:66455:0:(import.c:588:import_select_connection()) MGC192.168.204.126@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321618.657278:0:66455:0:(import.c:623:import_select_connection()) MGC192.168.204.126@tcp: import ffff8ce190885000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321618.657305:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578700 x1853163985726848/t0(0) o250->MGC192.168.204.126@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321618.657339:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7692417a-f5a3-4f69-bd37-096c76d4b4d8:7012:1853163985726848:0@lo:250:kworker.0 00000400:00000100:3.0:1767321618.657362:0:7012:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163985726848 offset 0 length 520: 4 00000100:00080000:3.0:1767321618.657388:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321618.657392:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321618.657395:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985725824:0@lo:38:kworker.0 00000100:00080000:3.0:1767321618.657403:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321618.657406:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321618.657409:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985725952:0@lo:38:kworker.0 00000100:00080000:3.0:1767321618.657415:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321618.657418:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321618.657421:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985726208:0@lo:38:kworker.0 00000100:00080000:3.0:1767321618.657429:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321618.657432:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321618.657435:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985726464:0@lo:38:kworker.0 00000100:00100000:2.0:1767321618.658250:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19e9c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985726592:0@lo:400:kworker.0 00000100:00100000:0.0:1767321618.658358:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a6a61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985726080:0@lo:400:kworker.0 00000100:00100000:1.0:1767321619.174106:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985726976:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321619.174247:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321619.176018:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985726976 00000100:00100000:3.0:1767321619.176036:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985726976 00000100:00100000:3.0:1767321619.176142:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3056 00000100:00100000:3.0:1767321619.176197:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985726976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321619.176242:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321619.176347:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985726976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 186us (2099us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321619.176356:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3056 00000100:00100000:2.0:1767321619.177054:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321619.177076:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985727104:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321619.177151:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321619.177873:0:59191:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985727104 00000100:00100000:0.0:1767321619.177882:0:59191:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985727104 00000100:00100000:0.0:1767321619.177955:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3057 00000100:00100000:0.0:1767321619.177964:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985727104:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321619.177996:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321619.178063:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985727104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (920us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321619.178070:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3057 00000100:00100000:1.0:1767321619.178642:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985726976:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321619.179083:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985727104:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321619.296717:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321620.317162:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321621.339657:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321622.371498:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321623.387582:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321623.726327:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321623.726471:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920683776 00000100:00100000:2.0:1767321623.726486:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920683776 00000100:00100000:2.0:1767321623.726551:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6849 00000100:00100000:2.0:1767321623.726561:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920683776:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321623.726592:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321623.726688:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b34d1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920683776:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 126us (374us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321623.726697:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6849 00000100:00100000:3.0:1767321623.727739:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321623.728228:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920683904 00000100:00100000:0.0:1767321623.728236:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920683904 00000100:00100000:0.0:1767321623.728320:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5932 00000100:00100000:0.0:1767321623.728330:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920683904:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321623.728416:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2516:x1853163920683904:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 86us (679us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321623.728423:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5932 00000100:00080000:3.0:1767321623.728865:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321623.772099:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321623.772111:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321623.772115:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321623.772120:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321623.772126:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321623.772137:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6866 00000100:00080000:0.0:1767321623.772142:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321623.772150:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321623.772220:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0380 x1853163985727232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321623.772256:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321623.772260:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321623.772262:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321623.772266:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6866 00000100:00080000:0.0:1767321623.772269:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321623.772272:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321623.772288:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a1f80 x1853163985727360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321623.772323:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321623.772327:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321623.772329:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321623.772333:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6866 00000100:00080000:0.0:1767321623.772336:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321623.772339:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321623.772355:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a2300 x1853163985727616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321623.772401:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321623.772405:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321623.772407:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321623.772411:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6866 00000100:00080000:0.0:1767321623.772413:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321623.772416:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:0.0:1767321623.772427:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1a55a0a80 x1853163985727872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321623.772541:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985727232:0@lo:38:kworker.0 00000100:00100000:2.0:1767321623.772633:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321623.772715:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985727360:0@lo:38:kworker.0 00000100:00100000:2.0:1767321623.772741:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321623.772766:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985727616:0@lo:38:kworker.0 00000100:00100000:2.0:1767321623.772788:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321623.772807:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985727872:0@lo:38:kworker.0 00000100:00100000:2.0:1767321623.772826:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321623.772862:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985727232 00000100:00100000:2.0:1767321623.772870:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985727232 00000100:00100000:2.0:1767321623.772902:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985727616 00000100:00100000:2.0:1767321623.772905:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985727616 00000100:00100000:2.0:1767321623.772918:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985727872 00000100:00100000:2.0:1767321623.772920:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985727872 00000100:00100000:2.0:1767321623.772937:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6850 00000100:00100000:2.0:1767321623.772945:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985727232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321623.772970:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321623.773015:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985727232:12345-0@lo:38:kworker.0 Request processed in 69us (382us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321623.773023:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6850 00000100:00100000:2.0:1767321623.773042:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6851 00000100:00100000:2.0:1767321623.773046:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985727616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321623.773055:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321623.773078:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985727616:12345-0@lo:38:kworker.0 Request processed in 30us (289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321623.773084:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6851 00000100:00100000:2.0:1767321623.773094:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6852 00000100:00100000:2.0:1767321623.773099:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985727872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321623.773108:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321623.773129:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7012:x1853163985727872:12345-0@lo:38:kworker.0 Request processed in 28us (301us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321623.773134:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6852 00000100:00080000:2.0:1767321623.773192:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321623.773198:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321623.773202:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985727232:0@lo:38:kworker.0 00000100:00080000:2.0:1767321623.773224:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321623.773227:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321623.773230:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985727616:0@lo:38:kworker.0 00000100:00080000:2.0:1767321623.773238:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321623.773241:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321623.773244:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985727872:0@lo:38:kworker.0 00000100:00100000:2.0:1767321623.773297:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985728128:0@lo:400:kworker.0 00000100:00100000:2.0:1767321623.773326:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321623.773595:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985727360 00000100:00100000:3.0:1767321623.773600:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985727360 00000100:00100000:3.0:1767321623.773628:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2203 00000100:00100000:3.0:1767321623.773633:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985727360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321623.773643:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321623.773680:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985727360:12345-0@lo:38:kworker.0 Request processed in 46us (937us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321623.773687:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2203 00000100:00100000:3.0:1767321623.773718:0:18370:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728128 00000100:00100000:3.0:1767321623.773722:0:18370:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728128 00000100:00100000:3.0:1767321623.773766:0:18370:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3821 00000100:00100000:3.0:1767321623.773773:0:18370:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985728128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.774198:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985727744:0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.774234:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321623.774259:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321623.774265:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985728000:0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.774287:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321623.774314:0:70202:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728000 00000100:00100000:0.0:1767321623.774317:0:70202:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728000 00000100:00100000:0.0:1767321623.774345:0:70202:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3822 00000100:00100000:0.0:1767321623.774352:0:70202:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985728000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.774401:0:70202:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_017:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985728000:12345-0@lo:400:kworker.0 Request processed in 50us (113us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321623.774407:0:70202:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3822 00000100:00100000:0.0:1767321623.774441:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985728000:0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.774554:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985727744 00000100:00100000:0.0:1767321623.774558:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985727744 00000100:00100000:0.0:1767321623.774586:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5933 00000100:00100000:0.0:1767321623.774593:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985727744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.774626:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985727744:12345-0@lo:400:kworker.0 Request processed in 34us (392us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321623.774632:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5933 00000100:00100000:0.0:1767321623.774812:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985727744:0@lo:400:kworker.0 00000100:00080000:0.0:1767321623.774861:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321623.774866:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321623.774868:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321623.774888:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321623.774893:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321623.774897:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985727360:0@lo:38:kworker.0 00000100:00100000:0.0:1767321623.774923:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985727488:0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.774947:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321623.774973:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985727488 00000100:00100000:0.0:1767321623.774977:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985727488 00000100:00100000:0.0:1767321623.775016:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5934 00000100:00100000:0.0:1767321623.775023:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985727488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321623.775054:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985727488:12345-0@lo:400:kworker.0 Request processed in 32us (106us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321623.775060:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5934 00000100:00080000:0.0:1767321623.775072:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321623.775085:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985727488:0@lo:400:kworker.0 00000100:00100000:3.0:1767321623.775260:0:18370:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985728128:12345-0@lo:400:kworker.0 Request processed in 1486us (1932us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321623.775266:0:18370:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3821 00000100:00100000:2.0:1767321623.777545:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985728128:0@lo:400:kworker.0 00000100:00100000:3.0:1767321624.283434:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321624.283551:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985728256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321624.283637:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321624.285601:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321624.285721:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985728384:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321624.285781:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321624.285885:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728256 00000100:00100000:3.0:1767321624.285896:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728256 00000100:00100000:3.0:1767321624.285984:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728384 00000100:00100000:3.0:1767321624.285987:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728384 00000100:00100000:0.0:1767321624.286020:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3058 00000100:00100000:0.0:1767321624.286033:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985728256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321624.286070:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321624.286135:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3579180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985728256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (2497us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321624.286145:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3058 00000100:00100000:0.0:1767321624.286186:0:59191:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3059 00000100:00100000:0.0:1767321624.286193:0:59191:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985728384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321624.286211:0:59191:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:0.0:1767321624.286242:0:59191:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a4df9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985728384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 50us (461us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321624.286248:0:59191:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3059 00000100:00100000:0.0:1767321624.286290:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985728384:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321624.286382:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a55a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985728256:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321624.411559:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321625.436142:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321626.460545:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321627.483415:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321628.509546:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321628.845123:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321628.845210:0:7008:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321628.845306:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920684160 00000100:00100000:0.0:1767321628.845316:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920684160 00000100:00100000:0.0:1767321628.845384:0:20685:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920684032 00000100:00100000:0.0:1767321628.845387:0:20685:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920684032 00000100:00100000:0.0:1767321628.845408:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5935 00000100:00100000:0.0:1767321628.845418:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920684160:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321628.845487:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920684160:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 71us (373us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321628.845495:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5935 00000100:00100000:0.0:1767321628.845523:0:20685:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6853 00000100:00100000:0.0:1767321628.845527:0:20685:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920684032:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321628.845550:0:20685:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321628.845581:0:20685:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2512:x1853163920684032:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 52us (371us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321628.845588:0:20685:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6853 00000100:00080000:2.0:1767321628.845619:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321628.892785:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321628.892795:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321628.892798:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321628.892802:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321628.892806:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321628.892813:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6872 00000100:00080000:3.0:1767321628.892818:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321628.892824:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321628.892859:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a300 x1853163985728512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321628.892884:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321628.892888:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321628.892889:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321628.892893:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6872 00000100:00080000:3.0:1767321628.892895:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321628.892898:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321628.892909:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985728640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321628.892952:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321628.892956:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321628.892958:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321628.892961:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6872 00000100:00080000:3.0:1767321628.892963:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321628.892966:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321628.892976:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a680 x1853163985728896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321628.893001:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321628.893004:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321628.893005:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321628.893008:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6872 00000100:00080000:3.0:1767321628.893010:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321628.893013:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321628.893021:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578380 x1853163985729152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321628.894248:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985729280:0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.894329:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321628.894410:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321628.894416:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985729024:0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.894438:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321628.894487:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729024 00000100:00100000:1.0:1767321628.894494:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729024 00000100:00100000:1.0:1767321628.894546:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5936 00000100:00100000:1.0:1767321628.894554:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985729024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.894604:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985729024:12345-0@lo:400:kworker.0 Request processed in 51us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321628.894611:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5936 00000100:00080000:1.0:1767321628.894630:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321628.894663:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985728768:0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.894684:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321628.894699:0:7016:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321628.894703:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985729408:0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.894721:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321628.894740:0:15468:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728768 00000100:00100000:1.0:1767321628.894744:0:15468:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728768 00000100:00100000:1.0:1767321628.894766:0:15468:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5937 00000100:00100000:1.0:1767321628.894772:0:15468:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985728768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.894800:0:15468:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7016:x1853163985728768:12345-0@lo:400:kworker.0 Request processed in 28us (115us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321628.894805:0:15468:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5937 00000100:00080000:1.0:1767321628.894816:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321628.894838:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7016:1853163985728768:0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.894858:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985729024:0@lo:400:kworker.0 00000100:00100000:0.0:1767321628.895101:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729280 00000100:00100000:0.0:1767321628.895106:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729280 00000100:00100000:0.0:1767321628.895143:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729408 00000100:00100000:0.0:1767321628.895146:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729408 00000100:00100000:0.0:1767321628.895182:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3823 00000100:00100000:0.0:1767321628.895188:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985729280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321628.895222:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dca4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985729280:12345-0@lo:400:kworker.0 Request processed in 35us (894us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321628.895227:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3823 00000100:00100000:0.0:1767321628.895238:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3824 00000100:00100000:0.0:1767321628.895243:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985729408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321628.895263:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985729408:12345-0@lo:400:kworker.0 Request processed in 20us (542us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321628.895267:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3824 00000100:00100000:0.0:1767321628.895303:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985728512:0@lo:38:kworker.0 00000100:00100000:0.0:1767321628.895326:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321628.895345:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985728640:0@lo:38:kworker.0 00000100:00100000:0.0:1767321628.895365:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321628.895383:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985728896:0@lo:38:kworker.0 00000100:00100000:0.0:1767321628.895400:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321628.895408:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985729152:0@lo:38:kworker.0 00000100:00100000:0.0:1767321628.895423:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321628.895458:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728640 00000100:00100000:0.0:1767321628.895461:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728640 00000100:00100000:0.0:1767321628.895477:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2204 00000100:00100000:0.0:1767321628.895480:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985728640:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321628.895498:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321628.895538:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985728640:12345-0@lo:38:kworker.0 Request processed in 56us (172us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321628.895543:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2204 00000100:00100000:0.0:1767321628.895557:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728512 00000100:00100000:0.0:1767321628.895569:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728512 00000100:00100000:0.0:1767321628.895584:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985728896 00000100:00100000:0.0:1767321628.895586:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985728896 00000100:00100000:0.0:1767321628.895594:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729152 00000100:00100000:0.0:1767321628.895596:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729152 00000100:00100000:0.0:1767321628.895609:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6854 00000100:00100000:0.0:1767321628.895612:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985728512:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321628.895621:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321628.895641:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985728512:12345-0@lo:38:kworker.0 Request processed in 28us (314us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321628.895646:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6854 00000100:00100000:0.0:1767321628.896026:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6855 00000100:00100000:0.0:1767321628.896031:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985728896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321628.896040:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321628.896067:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985728896:12345-0@lo:38:kworker.0 Request processed in 35us (667us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321628.896072:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6855 00000100:00100000:0.0:1767321628.896085:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6856 00000100:00100000:0.0:1767321628.896088:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985729152:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321628.896096:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321628.896115:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985729152:12345-0@lo:38:kworker.0 Request processed in 25us (691us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321628.896120:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6856 00000100:00080000:0.0:1767321628.896153:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321628.896158:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321628.896161:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985728512:0@lo:38:kworker.0 00000100:00080000:0.0:1767321628.896193:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321628.896196:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321628.896198:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985728640:0@lo:38:kworker.0 00000100:00080000:0.0:1767321628.896206:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321628.896208:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321628.896211:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985728896:0@lo:38:kworker.0 00000100:00080000:0.0:1767321628.896218:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321628.896221:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321628.896223:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985729152:0@lo:38:kworker.0 00000100:00080000:1.0:1767321628.899902:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321628.899913:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321628.899975:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985729408:0@lo:400:kworker.0 00000100:00100000:1.0:1767321628.900017:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985729280:0@lo:400:kworker.0 00000100:00100000:0.0:1767321629.403477:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321629.403501:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985729536:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321629.403588:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321629.403784:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729536 00000100:00100000:2.0:1767321629.403793:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729536 00000100:00100000:2.0:1767321629.403846:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3060 00000100:00100000:2.0:1767321629.403855:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985729536:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321629.403887:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321629.403937:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985729536:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (349us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321629.403945:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3060 00000100:00100000:2.0:1767321629.404209:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321629.404220:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985729664:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321629.404248:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321629.404354:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985729536:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321629.404453:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729664 00000100:00100000:3.0:1767321629.404458:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729664 00000100:00100000:3.0:1767321629.404492:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3061 00000100:00100000:3.0:1767321629.404499:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985729664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321629.404516:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321629.404569:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985729664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (320us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321629.404575:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3061 00000100:00100000:1.0:1767321629.404715:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985729664:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767321629.531982:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321630.560597:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321631.584545:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321632.605310:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321633.627397:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321633.967632:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321633.967815:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920684416 00000100:00100000:2.0:1767321633.967828:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920684416 00000100:00100000:2.0:1767321633.967911:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5938 00000100:00100000:2.0:1767321633.967924:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920684416:12345-192.168.204.26@tcp:400:kworker.0 00000100:00080000:1.0:1767321633.968080:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321633.984309:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3578380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2515:x1853163920684416:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 16371us (16668us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321633.984327:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5938 00000100:00100000:3.0:1767321633.984788:0:7007:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:2.0:1767321633.984957:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920684288 00000100:00100000:2.0:1767321633.984965:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920684288 00000100:00100000:2.0:1767321633.985003:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6857 00000100:00100000:2.0:1767321633.985009:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920684288:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:2.0:1767321633.985034:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321633.985091:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b357a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920684288:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 80us (304us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321633.985097:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6857 00000100:00080000:3.0:1767321634.011529:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321634.011543:0:66455:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321634.011546:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321634.011551:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321634.011556:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321634.011567:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6877 00000100:00080000:3.0:1767321634.011575:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321634.011582:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321634.011639:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3578000 x1853163985729792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321634.011668:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321634.011672:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321634.011675:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321634.011680:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6877 00000100:00080000:3.0:1767321634.011683:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321634.011688:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321634.011700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985729792:0@lo:38:kworker.0 00000100:00100000:3.0:1767321634.011706:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357a300 x1853163985729920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321634.011799:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767321634.011809:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321634.011811:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321634.011816:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321634.011822:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6877 00000100:00080000:3.0:1767321634.011825:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321634.011831:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:3.0:1767321634.011849:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b3579500 x1853163985730176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321634.011888:0:66455:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767321634.011891:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985729920:0@lo:38:kworker.0 00000100:00080000:3.0:1767321634.011893:0:66455:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321634.011898:0:66455:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321634.011905:0:66455:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6877 00000100:00080000:3.0:1767321634.011909:0:66455:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321634.011914:0:66455:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:2.0:1767321634.011928:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321634.011931:0:66455:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce1b357ad80 x1853163985730432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321634.011977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985730176:0@lo:38:kworker.0 00000100:00100000:3.0:1767321634.012010:0:7016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985730560:0@lo:400:kworker.0 00000100:00100000:2.0:1767321634.012023:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321634.012053:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985730432:0@lo:38:kworker.0 00000100:00100000:3.0:1767321634.012099:0:7016:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321634.012102:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321634.012218:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985730688:0@lo:400:kworker.0 00000100:00100000:1.0:1767321634.012342:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985730048:0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.026760:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321634.026888:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985730304:0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.026945:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321634.026991:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730560 00000100:00100000:0.0:1767321634.027001:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730560 00000100:00100000:0.0:1767321634.027071:0:70194:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730688 00000100:00100000:0.0:1767321634.027074:0:70194:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730688 00000100:00100000:0.0:1767321634.027099:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3825 00000100:00100000:0.0:1767321634.027109:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985730560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.027214:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7016:x1853163985730560:12345-0@lo:400:kworker.0 Request processed in 105us (15125us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321634.027222:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3825 00000100:00100000:0.0:1767321634.027245:0:70194:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3826 00000100:00100000:0.0:1767321634.027252:0:70194:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985730688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.027287:0:70194:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985730688:12345-0@lo:400:kworker.0 Request processed in 36us (541us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321634.027293:0:70194:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3826 00000100:00080000:0.0:1767321634.027310:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321634.027315:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321634.027336:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729792 00000100:00100000:0.0:1767321634.027340:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729792 00000100:00100000:0.0:1767321634.027360:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730176 00000100:00100000:0.0:1767321634.027363:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730176 00000100:00100000:0.0:1767321634.027376:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730432 00000100:00100000:0.0:1767321634.027379:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730432 00000100:00100000:0.0:1767321634.027388:0:71760:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730304 00000100:00100000:0.0:1767321634.027391:0:71760:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730304 00000100:00100000:0.0:1767321634.027424:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5939 00000100:00100000:0.0:1767321634.027430:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985730304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.027463:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:7015:x1853163985730304:12345-0@lo:400:kworker.0 Request processed in 33us (518us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321634.031718:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5939 00000100:00100000:0.0:1767321634.031780:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6858 00000100:00100000:0.0:1767321634.032651:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985729792:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321634.032693:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321634.032870:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985729792:12345-0@lo:38:kworker.0 Request processed in 1080us (21067us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321634.032880:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6858 00000100:00100000:0.0:1767321634.032910:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6859 00000100:00100000:0.0:1767321634.032916:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985730176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321634.032927:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321634.032955:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985730176:12345-0@lo:38:kworker.0 Request processed in 37us (20942us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321634.032960:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6859 00000100:00100000:0.0:1767321634.032970:0:71760:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6860 00000100:00100000:0.0:1767321634.032975:0:71760:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985730432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321634.032985:0:71760:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321634.033008:0:71760:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:0+-99:7012:x1853163985730432:12345-0@lo:38:kworker.0 Request processed in 31us (20918us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321634.033013:0:71760:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6860 00000100:00080000:0.0:1767321634.033061:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321634.033117:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7015:1853163985730304:0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.033376:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985730688:0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.035091:0:15472:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985729920 00000100:00100000:0.0:1767321634.035109:0:15472:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985729920 00000100:00100000:0.0:1767321634.035191:0:15472:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2205 00000100:00100000:0.0:1767321634.035200:0:15472:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985729920:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321634.035227:0:15472:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321634.035318:0:15472:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:0+-99:7012:x1853163985729920:12345-0@lo:38:kworker.0 Request processed in 117us (23390us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321634.035326:0:15472:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2205 00000100:00100000:3.0:1767321634.043932:0:7016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a61f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7016:1853163985730560:0@lo:400:kworker.0 00000100:00100000:1.0:1767321634.046376:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321634.046838:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730048 00000100:00100000:0.0:1767321634.046852:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730048 00000100:00080000:1.0:1767321634.046939:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321634.046950:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5940 00000100:00100000:0.0:1767321634.046963:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985730048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321634.047055:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcd9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163985730048:12345-0@lo:400:kworker.0 Request processed in 95us (691us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321634.047063:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5940 00000100:00100000:1.0:1767321634.047237:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163985730048:0@lo:400:kworker.0 00000100:00080000:2.0:1767321634.047951:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321634.047964:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321634.047969:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985730432:0@lo:38:kworker.0 00000100:00080000:2.0:1767321634.047983:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321634.047987:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321634.047990:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b357a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985729920:0@lo:38:kworker.0 00000100:00080000:2.0:1767321634.048000:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321634.048003:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321634.048006:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3579500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985730176:0@lo:38:kworker.0 00000100:00080000:2.0:1767321634.048015:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321634.048017:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321634.048019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3578000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985729792:0@lo:38:kworker.0 00000100:00100000:1.0:1767321634.523730:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985730816:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321634.523835:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321634.524149:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730816 00000100:00100000:2.0:1767321634.524194:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730816 00000100:00100000:2.0:1767321634.524271:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3062 00000100:00100000:2.0:1767321634.524282:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985730816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321634.524316:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321634.524767:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19dcdb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985730816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 485us (929us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321634.524776:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3062 00000100:00100000:1.0:1767321634.524890:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985730816:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321634.526109:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985730944:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321634.526238:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321634.527391:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985730944 00000100:00100000:2.0:1767321634.527406:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985730944 00000100:00100000:2.0:1767321634.527480:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3063 00000100:00100000:2.0:1767321634.527489:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985730944:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321634.527520:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321634.527605:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985730944:12345-0@lo:13:osp-pre-0-1.0 Request processed in 118us (1367us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321634.527613:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3063 00000100:00100000:0.0:1767321634.528785:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3519500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985730944:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767321634.651443:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321635.682376:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321636.699735:0:15461:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321637.723378:0:15460:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321638.747366:0:27416:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321639.086261:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:3.0:1767321639.086985:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.26@tcp (source: 12345-192.168.204.26@tcp) 00000100:00100000:0.0:1767321639.087114:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920684672 00000100:00100000:0.0:1767321639.087124:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920684672 00000100:00100000:0.0:1767321639.087235:0:15466:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163920684544 00000100:00100000:0.0:1767321639.087238:0:15466:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163920684544 00000100:00100000:0.0:1767321639.087256:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 5941 00000100:00100000:0.0:1767321639.087265:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920684672:12345-192.168.204.26@tcp:400:kworker.0 00000100:00100000:0.0:1767321639.087356:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b3519180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:5a08f414-1531-480f-9dc1-e0b5abd26116+5:2513:x1853163920684672:12345-192.168.204.26@tcp:400:kworker.0 Request processed in 92us (1106us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321639.087363:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 5941 00000100:00100000:0.0:1767321639.087381:0:15466:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.26@tcp, seq: 6861 00000100:00100000:0.0:1767321639.087384:0:15466:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920684544:12345-192.168.204.26@tcp:38:kworker.0 00010000:00020000:0.0:1767321639.087404:0:15466:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.26@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321639.087425:0:15466:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1b351a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2512:x1853163920684544:12345-192.168.204.26@tcp:38:kworker.0 Request processed in 40us (443us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321639.087430:0:15466:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.26@tcp, seq: 6861 00000100:00080000:2.0:1767321639.088684:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321639.131404:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321639.131415:0:44026:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7692417a-f5a3-4f69-bd37-096c76d4b4d8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321639.131418:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321639.131422:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321639.131426:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321639.131434:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6882 00000100:00080000:1.0:1767321639.131438:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321639.131444:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff8ce1b464d000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321639.131482:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd8000 x1853163985731072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321639.131512:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321639.131516:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321639.131517:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321639.131521:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6882 00000100:00080000:1.0:1767321639.131524:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321639.131527:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff8ce1b47d4000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321639.131539:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcda300 x1853163985731200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321639.131575:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321639.131578:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321639.131579:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321639.131583:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6882 00000100:00080000:1.0:1767321639.131586:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321639.131589:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff8ce1be614000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321639.131603:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcd8e00 x1853163985731456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321639.131623:0:44026:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321639.131626:0:44026:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321639.131628:0:44026:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321639.131631:0:44026:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6882 00000100:00080000:1.0:1767321639.131633:0:44026:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321639.131636:0:44026:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff8ce19fe7e000 using connection 192.168.204.126@tcp/0@lo 00000100:00100000:1.0:1767321639.131646:0:44026:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8ce19dcda680 x1853163985731712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321639.131716:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985731968:0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.131774:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321639.131831:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321639.131837:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985731328:0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.131857:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321639.131889:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731328 00000100:00100000:1.0:1767321639.131895:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731328 00000100:00100000:1.0:1767321639.131937:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5942 00000100:00100000:1.0:1767321639.131944:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985731328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.131987:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7015:x1853163985731328:12345-0@lo:400:kworker.0 Request processed in 42us (128us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321639.131993:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5942 00000100:00100000:1.0:1767321639.132013:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985731840:0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132045:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321639.132062:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321639.132066:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985731584:0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132082:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321639.132115:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731968 00000100:00100000:1.0:1767321639.132119:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731968 00000100:00100000:1.0:1767321639.132142:0:70193:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731840 00000100:00100000:1.0:1767321639.132144:0:70193:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731840 00000100:00100000:1.0:1767321639.132160:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3827 00000100:00100000:1.0:1767321639.132195:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985731968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132224:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985731968:12345-0@lo:400:kworker.0 Request processed in 31us (450us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321639.132230:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3827 00000100:00100000:1.0:1767321639.132242:0:70193:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3828 00000100:00100000:1.0:1767321639.132248:0:70193:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985731840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132277:0:70193:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163985731840:12345-0@lo:400:kworker.0 Request processed in 29us (230us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321639.132282:0:70193:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3828 00000100:00100000:1.0:1767321639.132303:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731584 00000100:00100000:1.0:1767321639.132306:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731584 00000100:00100000:1.0:1767321639.132326:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5943 00000100:00100000:1.0:1767321639.132331:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985731584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132354:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a54b6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163985731584:12345-0@lo:400:kworker.0 Request processed in 23us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321639.132359:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5943 00000100:00100000:1.0:1767321639.132392:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163985731584:0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132401:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7014:1853163985731840:0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132423:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7015:1853163985731328:0@lo:400:kworker.0 00000100:00100000:1.0:1767321639.132430:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1a54b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985731968:0@lo:400:kworker.0 00000100:00080000:2.0:1767321639.134155:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321639.134186:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321639.134189:0:15488:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321639.134210:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985731072:0@lo:38:kworker.0 00000100:00100000:2.0:1767321639.134262:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321639.134296:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985731200:0@lo:38:kworker.0 00000100:00100000:2.0:1767321639.134319:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321639.134347:0:16142:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731200 00000100:00100000:2.0:1767321639.134351:0:16142:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731200 00000100:00100000:2.0:1767321639.134387:0:16142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2206 00000100:00100000:2.0:1767321639.134394:0:16142:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985731200:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321639.134411:0:16142:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321639.134442:0:16142:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:0+-99:7012:x1853163985731200:12345-0@lo:38:kworker.0 Request processed in 47us (121us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321639.134451:0:16142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2206 00000100:00080000:2.0:1767321639.134484:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b47d4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321639.134488:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321639.134491:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7012:1853163985731200:0@lo:38:kworker.0 00000100:00100000:2.0:1767321639.134498:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985731456:0@lo:38:kworker.0 00000100:00100000:2.0:1767321639.134518:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321639.134539:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731072 00000100:00100000:2.0:1767321639.134542:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731072 00000100:00100000:2.0:1767321639.134556:0:15467:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731456 00000100:00100000:2.0:1767321639.134558:0:15467:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731456 00000100:00100000:2.0:1767321639.134571:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6862 00000100:00100000:2.0:1767321639.134575:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985731072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321639.134585:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321639.134606:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985731072:12345-0@lo:38:kworker.0 Request processed in 30us (343us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321639.134611:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6862 00000100:00100000:2.0:1767321639.134621:0:15467:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6863 00000100:00100000:2.0:1767321639.134624:0:15467:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985731456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321639.134632:0:15467:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321639.134652:0:15467:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7012:x1853163985731456:12345-0@lo:38:kworker.0 Request processed in 26us (133us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321639.134656:0:15467:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6863 00000100:00080000:2.0:1767321639.134674:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1be614000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321639.134677:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321639.134680:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7012:1853163985731456:0@lo:38:kworker.0 00000100:00100000:2.0:1767321639.134686:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985731712:0@lo:38:kworker.0 00000100:00100000:2.0:1767321639.134984:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321639.135144:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce1b464d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321639.135148:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321639.135151:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7012:1853163985731072:0@lo:38:kworker.0 00000100:00100000:1.0:1767321639.135222:0:26020:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985731712 00000100:00100000:1.0:1767321639.135226:0:26020:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985731712 00000100:00100000:1.0:1767321639.135253:0:26020:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 6864 00000100:00100000:1.0:1767321639.135259:0:26020:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985731712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321639.135270:0:26020:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321639.135299:0:26020:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce19e9c5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7012:x1853163985731712:12345-0@lo:38:kworker.0 Request processed in 40us (315us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321639.135305:0:26020:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 6864 00000100:00080000:2.0:1767321639.135865:0:7012:0:(import.c:65:import_set_state_nolock()) ffff8ce19fe7e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321639.135869:0:7012:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321639.135872:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce19dcda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7012:1853163985731712:0@lo:38:kworker.0 00000100:00100000:2.0:1767321639.644020:0:7015:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321639.644043:0:7015:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985732096:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321639.644118:0:7015:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321639.644327:0:27399:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985732096 00000100:00100000:3.0:1767321639.644336:0:27399:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985732096 00000100:00100000:3.0:1767321639.644390:0:27399:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3064 00000100:00100000:3.0:1767321639.644399:0:27399:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985732096:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321639.644425:0:27399:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:3.0:1767321639.644472:0:27399:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a6a61500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7015:x1853163985732096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (353us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321639.644479:0:27399:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3064 00000100:00100000:0.0:1767321639.644590:0:7015:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b3518380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7015:1853163985732096:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321639.647289:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985732224:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321639.647353:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321639.647451:0:18373:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163985732224 00000100:00100000:2.0:1767321639.647459:0:18373:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163985732224 00000100:00100000:2.0:1767321639.647506:0:18373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3065 00000100:00100000:2.0:1767321639.647516:0:18373:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985732224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321639.647545:0:18373:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 254912 pending 0 free 3923529728 avail 3692007424 00000100:00100000:2.0:1767321639.647591:0:18373:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8ce1a55a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163985732224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (238us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321639.647598:0:18373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3065 00000100:00100000:0.0:1767321639.648189:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8ce1b351a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7013:1853163985732224:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321639.773886:0:32313:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767321640.795753:0:27417:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321641.819426:0:32312:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 93732 lines, 93732 kept, 0 dropped, 0 bad.